// Isis2 System, V2.0.$Rev: 1306 $, Developed by Kenneth P. Birman, (c) 2010, 2011, 2012, 2013. All rights reserved.
//       This code is subject to copyright and other intellectual property restrictions and
//       may be used only under license from Dr. Birman or his designated agents.
//
// It is a violation of international intellectual property protection laws to remove this notice.
//
// Dr. Birman has elected to grant Cornell University and other researchers the right to experiment with Isis2 and
// to create a derived version Isis2 V2.0.000 (and other future versions, using the same version numbering scheme)
//
// Dr. Birman and Cornell University intend to place Isis2 V1.x.xxxx and V2.x.xxxx into the public domain, in the
// manner described below, and subject to licensing provisions intended to protect Dr. Birman and Cornell from
// any responsibility for consequences of the use of this technology, even when used precisely as intended.
// Isis2 is a free research prototype and not a product, and the user of the system accepts the responsibilities
// and consequences of this research prototype status as an implication of his or her decision to use the system.
//
// Licensing terms for Isis2 v1.x.xxxx and v2.x.xxxx use freeBSD license language and include indemnification 
// whereby the end user holds Dr. Birman and Cornell harmless for any and all uses of this technology.  Dr.
// Birman and Cornell University are not aware of any external patents, copyrights or trademarks
// that might be required by users of the Isis2 system, but have not conducted a
// thorough patent search and are not able to guarantee that no such licenses exist.  The end user accepts
// full responsibility to obtain any needed licenses required for the conduct of their work with this
// system, and holds Cornell harmless even in the event that Isis2 itself is found to infringe some
// patent, copyright or trademark.  Cornell holds no patents that read on the Isis2 technology and is not
// planning to seek patents on this work, nor is Dr. Birman personally.  Indeed, because of the extensive
// track record of publications in this area over a period of some 30 years, it seems unlikely that patents
// could be obtained on the technology by any party.  Nonetheless, Cornell (and Dr. Birman) cannot guarantee
// that this technology doesn't infringe any existing or future patent.  Should licenses be required on any
// existing or future technology, the user of Isis2, and not Dr. Birman or Cornell, has sole responsibility
// to negotiate such licenses and to pay any associated fees.
//
// Source code access may, in some cases, be granted to facilitate code maintanence.  Contact Dr. Birman
// for details.
//
// Dr. Birman and Cornell assert no ownership interest of any kind in applications developed using Isis2,
// as distinct from the Isis2 system per-se.  Example applications included in the Isis2 documentation may be
// copied, modified and incorporated into end-user applications without limitation or restriction: they are
// treated as public domain material.
//
// At the present time, Dr. Birman and his group at Cornell are providing support for this technology.  No
// committment, implied or explicit, is made to resolve any particular issue or to fix any particular bug.
// Like any complex technology, the user should expect Isis2 to have some bugs, perhaps serious, and should
// be aware that precisely because this is a free technology, support may be slow, frustrating, and may
// not lead to a successful resolution of the issue.  Isis2 is not a product, and there are no current plans
// to create a commercial product in this space.
//
// Dr. Birman reserves version numbers in the v1.x.xxxx range for versions of the system created by him
// privately, outside of his Cornell employment.  Isis2 versions in the v2.x.xxxx range are reserved
// for binary and code releases by the Cornell group headed by Dr. Birman and Dr. Robbert van Renesse.
//
// Please contact Dr. Birman at 607-255-9199 if you have any questions about the use of this technology.
// His email is ken@cs.cornell.edu and his web site is http://www.cs.cornell.edu/ken
//
// *******************************************************************************************
// A few comments on coding conventions used here
//       First, this code is extremely multithreaded and the threading and protection logic is key to correctness
//             Don't understand that logic?  Don't touch this code!
//       The C# lock statement didn't mix well with my use of thread priorities; use 
//                                       using(LockAndElevate(lock-object)) { ... }
//       Isis has its own locking infrastructure for coarse-grained long-lived locks; use these for that sort of thing
// ********************************************************************************************

//#define ___ANDROID__                     // In principle, should be automatically set, needed if compiling on Mono for Android
//#define __MonoCS__                       // Similarly, should be automatically set, needed when compiling for C# on Linux via Mono

// The DEFINEs that follow are useful in debugging but not for production runs.  Some can be VERY slow
#define TRACKLOCKINFO                   // If defined, tracks lock information, warns about apparent deadlocks or priority inversions
//#define TRACKLOCKINGORDER               // If both are defined, watches for potential lock ordering issues (e.g. usually any single thread locks A before B, but now B was locked, then A).  Broken in .NET4
//#define EXTRACTCALLSTACKS               // A risky and expensive .NET mechanism that can extract the call stack but sometimes seems to trigger deadlocks or other exceptions
//#define NONLOCALSTACKTRACES             // If defined, Isis will sometimes print stack traces of seemingly stuck threads but for this uses a deprecated API and sometimes malfunctions (which is why they deprecated it!)
//#define WARN_ON_LONG_DELAYS             // Useful if locking problems are suspected in message handlers

// Because Debug is a constant, we disable what would otherwise be 1000's of warnings
#pragma warning disable 0162
#pragma warning disable 0429

using System;
using System.IO;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Net;
using System.Net.Sockets;
using System.Net.NetworkInformation;
using System.Diagnostics;
using System.Reflection;
using System.Runtime.Serialization.Formatters.Binary;
using System.Security.Cryptography;
using System.Text;
using System.IO.MemoryMappedFiles;

namespace Isis
{
    // The fatal exceptions also cause Isis to shut down
    /// <summary>
    /// General purpose exception
    /// </summary>
    [Serializable]
    public class IsisException : Exception
    {
        /// <summary>
        /// General purpose exception constructor
        /// </summary>
        /// <param name="s">Reason for the exception</param>
        public IsisException(string s)
            : base(s)
        {
            if (!IsisSystem.IsisActive && IsisSystem.IsisWasActive)
                return;
            IsisSystem.WriteAckInfo();
            IsisSystem.Shutdown(s);
        }
    }
    /// <summary>
    /// General purpose Client exception
    /// </summary>
    [Serializable]
    public class IsisClientException : Exception
    {
        /// <summary>
        /// General purpose exception constructor
        /// </summary>
        /// <param name="s">Reason for the exception</param>
        public IsisClientException(string s)
            : base(s)
        {
        }
    }
    /// <summary>
    /// General purpose DHT exception
    /// </summary>
    [Serializable]
    public class IsisDHTException : Exception
    {
        /// <summary>
        /// General purpose exception constructor
        /// </summary>
        /// <param name="s">Reason for the exception</param>
        public IsisDHTException(string s)
            : base(s)
        {
        }
    }
    /// <summary>
    /// Thrown by disklogger when SafeSend is employed without first setting the SafeSendThreshold, or
    /// if the number of members of the group isn't at least SafeSendThreshold when multicasts are sent
    /// </summary>
    [Serializable]
    public class SafeSendException : Exception
    {
        /// <summary>
        /// General purpose exception constructor
        /// </summary>
        /// <param name="s">Reason for the exception</param>
        public SafeSendException(string s)
            : base(s)
        {
            IsisSystem.WriteAckInfo(); IsisSystem.Shutdown(s);
        }
    }
    // The fatal exceptions also cause Isis to shut down
    /// <summary>
    /// General purpose exception
    /// </summary>
    [Serializable]
    public class RejectedMessageException : Exception
    {
        /// <summary>
        /// Message rejected by Isis because of a signature problem
        /// </summary>
        /// <param name="s">Reason for the exception</param>
        public RejectedMessageException(string s)
            : base(s)
        {
        }
    }
    /// <summary>
    /// Aggregation failure exception
    /// </summary>
    [Serializable]
    public class AggregationFailed : Exception
    {
        /// <summary>
        /// Aggregaton failure exception constructor
        /// </summary>
        public AggregationFailed(int i)
        {
        }
    }

    /// <summary>
    ///Thrown if an AbortReply is received
    /// </summary>
    [Serializable]
    public class IsisAbortReplyException : Exception
    {
        /// <summary>
        /// Constructor for AbortReply exceptions
        /// </summary>
        /// <param name="s"></param>
        public IsisAbortReplyException(string s) : base(s) { }
    }

    /// <summary>
    ///Thrown if a SafeSend can't complete because the group has fewer than SafeSendThreshold members
    /// </summary>
    [Serializable]
    public class IsisSafeSendException : Exception
    {
        /// <summary>
        /// Constructor for AbortReply exceptions
        /// </summary>
        /// <param name="s"></param>
        public IsisSafeSendException(string s) : base(s) { }
    }

    [Serializable]
    class MCMDException : Exception { public MCMDException(string s) : base(s) { IsisSystem.Shutdown(s); } }
    /// <summary>
    /// Thrown if a Join-only operation can't find a preexisting group with the right name or address
    /// </summary>
    [Serializable]
    public class GroupNotFound : Exception
    {
        /// <summary>
        /// Constructor for GroupNotFound exceptions
        /// </summary>
        /// <param name="s">Reason for the problem</param>
        public GroupNotFound(string s) : base(s) { IsisSystem.Shutdown(s); }
    }

    /// <summary>
    /// Thrown when Isis is shutting down
    /// </summary>
    [Serializable]
    public class IsisShutdown : Exception
    {
        /// <summary>
        /// Constructor for IsisShutdown exceptions
        /// </summary>
        /// <param name="s">Reason Isis shut itself down</param>
        public IsisShutdown(string s)
            : base(s)
        {
            if (IsisSystem.shuttingDown)
                Environment.Exit(0);       // If we start to throw additional exceptions when attempting to shut down gracefully, give up
            IsisSystem.Shutdown();
        }
    }

    /// <summary>
    /// Type signature for an Isis ViewHandler callback method
    /// </summary>
    /// <param name="v"></param>
    public delegate void ViewHandler(View v);

    /// <summary>
    /// Type signature for an Isis universal callback method
    /// </summary>
    /// <param name="objs"></param>
    public delegate void UCallback(object[] objs);

    /// <summary>
    /// Type signature for an Isis CheckPtMaker callback method.  Creates new checkpoints
    /// </summary>
    /// <param name="v"></param>
    public delegate void ChkptMaker(View v);
    /// <summary>
    /// If defined, returns true if this member will send the checkpoint to the joiners, and false if not.
    /// </summary>
    /// <param name="v">The new view of the group</param>
    /// <param name="who">A specific joiner for whom the question is being posed</param>
    /// <returns>True if the member in which the call was done is responsible for sending the checkpoint for the specified joiner.</returns>
    /// <remarks>By default, when a new view is defined all members evaluate v.IAmLeader() and the (single) leader creates the checkpoint used to initialize (all) the joiners.
    /// That is, a single checkpoint is made, by a single member, and a copy is delivered to each joiner.  However, you can override this behavior if desired.
    /// When you do so, by defining the ChkptChoser(), Isis does a parallel evaluation of this method in every current group member.  For each joiner, a single
    /// member will be picked to send a checkpoint to it.  All others must return false.  If nobody returns true, the joiner will hang waiting for
    /// a checkpoint that will never arrive.  If a failure disrupts the checkpoint transfer, the joiner will throw a "Join failed" exception.
    /// </remarks>
    public delegate bool ChkptChoser(View v, Address who);
    /// <summary>
    /// Type signature for an Isis group initializer callback method
    /// </summary>
    public delegate void Initializer();
    /// <summary>
    /// Type signature for an Isis Timer callback handler method
    /// </summary>
    public delegate void TimerCallback();

    /// <summary>
    /// Type signature for an Isis Watch callback handler method
    /// </summary>
    /// <param name="ev">The event (W_JOIN or W_LEAVE)</param>
    public delegate void Watcher(int ev);

    /// <summary>
    /// Signature for the callback in a master when a new worker registers via RunAsWorker()
    /// </summary>
    /// <param name="who"></param>
    public delegate void NewWorker(Address who);

    /// <summary>
    /// Type signature for a SafeSend logging method, used to ensure the durability of SafeSend multicasts
    /// </summary>
    /// <param name="m">A pending SafeSend message, not yet committed for delivery</param>
    public delegate void durabilityMethod(Msg m);

    /// <summary>
    /// Callback on broken lock
    /// </summary>
    /// <param name="why">LOCK_TRANSFER (to rank 0 member) or LOCK_BROKEN</param>
    /// <param name="name">Lock name</param>
    /// <param name="holder">Previous lock holder</param>
    public delegate void LockBroken(int why, string name, Address holder);

    /// <summary>
    /// Signature for the Isis logging callback
    /// </summary>
    /// <param name="LEvent">Event types, as defined in Group</param>
    /// <param name="StartorDone">IL_START or IL_DONE</param>
    /// <param name="sender">The process that initiated this operation</param>
    /// <param name="LId">An identifier to match START and DONE events and track queries</param>
    /// <param name="args">Parameters for this type of event</param>
    public delegate void ILFunc(int LEvent, int StartorDone, Address sender, long LId, params object[] args);

    internal class Callable
    {
        internal int nParams;
        internal Type[] ptypes;
        internal Delegate hisCb;
        internal Delegate cb;

        internal Callable(Delegate hisCb)
        {
            this.hisCb = hisCb;
            MethodInfo mi = hisCb.GetType().GetMethod("Invoke");
            ParameterInfo[] pi = mi.GetParameters();
            ptypes = pi.Select(p => p.ParameterType).ToArray();
            nParams = ptypes.Length;
#if !(__MonoCS__  || ___ANDROID__) //VN
            try
            {
                if (nParams == 0)
                    cb = Delegate.CreateDelegate(typeof(Action), hisCb.Target, hisCb.Method, false);
                else if (nParams <= 16)
                {
                    cb = Delegate.CreateDelegate(System.Linq.Expressions.Expression.GetActionType(ptypes), hisCb.Target, hisCb.Method, false);
                    if (cb == null)
                    {
                        Type rtype = hisCb.GetType().GetMethod("Invoke").ReturnType;
                        if (rtype != null)
                        {
                            // Callback to a function method; used in the DHTPutCollisionResolver logic
                            Isis.ArrayResize<Type>(ref ptypes, ptypes.Length + 1);
                            ptypes[ptypes.Length - 1] = rtype;
                            cb = Delegate.CreateDelegate(System.Linq.Expressions.Expression.GetFuncType(ptypes), hisCb.Target, hisCb.Method, false);
                        }
                    }
                }
            }
            // If we get a "MethodInfo must be a runtime MethodInfo object." error, use old way, otherwise propegate exception
            catch (ArgumentException e)
            {
                if (e.ParamName == null || !e.ParamName.Equals("method"))
                    throw;
            }
#else
            cb = null;
#endif
        }

        internal void doUpcall(object[] args)
        {
            if (args.Length != nParams)
                throw new ArgumentException("Argument count must match number of parameters");
#if !(__MonoCS__ || ___ANDROID__) //VN
            if (cb != null)
                switch (nParams)
                {
                    case 0:
                        ((dynamic)cb).Invoke();
                        break;
                    case 1:
                        ((dynamic)cb).Invoke((dynamic)args[0]);
                        break;
                    case 2:
                        ((dynamic)cb).Invoke((dynamic)args[0], (dynamic)args[1]);
                        break;
                    case 3:
                        ((dynamic)cb).Invoke((dynamic)args[0], (dynamic)args[1], (dynamic)args[2]);
                        break;
                    case 4:
                        ((dynamic)cb).Invoke((dynamic)args[0], (dynamic)args[1], (dynamic)args[2], (dynamic)args[3]);
                        break;
                    case 5:
                        ((dynamic)cb).Invoke((dynamic)args[0], (dynamic)args[1], (dynamic)args[2], (dynamic)args[3], (dynamic)args[4]);
                        break;
                    case 6:
                        ((dynamic)cb).Invoke((dynamic)args[0], (dynamic)args[1], (dynamic)args[2], (dynamic)args[3], (dynamic)args[4], (dynamic)args[5]);
                        break;
                    case 7:
                        ((dynamic)cb).Invoke((dynamic)args[0], (dynamic)args[1], (dynamic)args[2], (dynamic)args[3], (dynamic)args[4], (dynamic)args[5], (dynamic)args[6]);
                        break;
                    case 8:
                        ((dynamic)cb).Invoke((dynamic)args[0], (dynamic)args[1], (dynamic)args[2], (dynamic)args[3], (dynamic)args[4], (dynamic)args[5], (dynamic)args[6], (dynamic)args[7]);
                        break;
                    case 9:
                        ((dynamic)cb).Invoke((dynamic)args[0], (dynamic)args[1], (dynamic)args[2], (dynamic)args[3], (dynamic)args[4], (dynamic)args[5], (dynamic)args[6], (dynamic)args[7], (dynamic)args[8]);
                        break;
                    case 10:
                        ((dynamic)cb).Invoke((dynamic)args[0], (dynamic)args[1], (dynamic)args[2], (dynamic)args[3], (dynamic)args[4], (dynamic)args[5], (dynamic)args[6], (dynamic)args[7], (dynamic)args[8], (dynamic)args[9]);
                        break;
                    case 11:
                        ((dynamic)cb).Invoke((dynamic)args[0], (dynamic)args[1], (dynamic)args[2], (dynamic)args[3], (dynamic)args[4], (dynamic)args[5], (dynamic)args[6], (dynamic)args[7], (dynamic)args[8], (dynamic)args[9], (dynamic)args[10]);
                        break;
                    case 12:
                        ((dynamic)cb).Invoke((dynamic)args[0], (dynamic)args[1], (dynamic)args[2], (dynamic)args[3], (dynamic)args[4], (dynamic)args[5], (dynamic)args[6], (dynamic)args[7], (dynamic)args[8], (dynamic)args[9], (dynamic)args[10], (dynamic)args[1]);
                        break;
                    case 13:
                        ((dynamic)cb).Invoke((dynamic)args[0], (dynamic)args[1], (dynamic)args[2], (dynamic)args[3], (dynamic)args[4], (dynamic)args[5], (dynamic)args[6], (dynamic)args[7], (dynamic)args[8], (dynamic)args[9], (dynamic)args[10], (dynamic)args[11], (dynamic)args[12]);
                        break;
                    case 14:
                        ((dynamic)cb).Invoke((dynamic)args[0], (dynamic)args[1], (dynamic)args[2], (dynamic)args[3], (dynamic)args[4], (dynamic)args[5], (dynamic)args[6], (dynamic)args[7], (dynamic)args[8], (dynamic)args[9], (dynamic)args[10], (dynamic)args[11], (dynamic)args[12], (dynamic)args[13]);
                        break;
                    case 15:
                        ((dynamic)cb).Invoke((dynamic)args[0], (dynamic)args[1], (dynamic)args[2], (dynamic)args[3], (dynamic)args[4], (dynamic)args[5], (dynamic)args[6], (dynamic)args[7], (dynamic)args[8], (dynamic)args[9], (dynamic)args[10], (dynamic)args[11], (dynamic)args[12], (dynamic)args[13], (dynamic)args[14]);
                        break;
                }
            else
            {
                MethodInfo mi = hisCb.GetType().GetMethod("Invoke");
                mi.Invoke(hisCb, args);
            }
#else
            hisCb.DynamicInvoke(args);
#endif
        }
    }

    /// <summary>
    /// Designates a class as suitable for automatic marshalling via Isis.
    /// </summary>
    /// <remarks>
    /// When the <bf>Automarshalled</bf> attribute is specified, Isis will automatically marshall the fields of 
    /// a class into a byte[] array, and create and initialize object instances from received byte[] arrays.
    /// </remarks>
    [AttributeUsage(AttributeTargets.All)]
    public class AutoMarshalled : Attribute
    {
        /// <summary>
        /// AutoMarshalled has no parameters
        /// </summary>
        public AutoMarshalled()
        {
        }
    }

    /// <summary>
    /// Inteface defintion for Isis aggregators
    /// </summary>
    /// <remarks>
    /// To define a custom Aggregator, implement the IAggregator interface, create an instance for each group in which you will use it,
    /// and register that instance via Group.RegisterAggregator.  If you need to have multiple instances of one aggregator
    /// for a single group, encode an instance identifier of some kind into the key; Isis won't allow you to register more than one
    /// aggregator of a single type in a single group.
    /// </remarks>
    /// <typeparam name="KeyType">The KeyType must be a primitive C# base type, or must implement the IEqualityComparer interface</typeparam>
    /// <typeparam name="ValueType"></typeparam>
    public interface IAggregator<KeyType, ValueType>
    {
        /// <summary>
        /// Combines two objects of type ValueType, which are associated with the given key.  Canned from an Isis thread.
        /// </summary>
        /// <param name="key">The key associated with this aggregation operation</param>
        /// <param name="fromLeft">Value received from the node to my left on this token ring</param>
        /// <param name="fromBelow">Value coming up from below</param>
        /// <returns></returns>
        ValueType Aggregate(KeyType key, ValueType fromLeft, ValueType fromBelow);
    }

    /// <summary>
    /// AggregatorDel is a delegate type that can be used to do in-line anonymous declaration of Isis aggregators in calls to Group.RegisterAggregator
    /// </summary>
    /// <typeparam name="KType">The associated KeyType</typeparam>
    /// <typeparam name="Vtype">The associated ValueType</typeparam>
    /// <param name="key">The key</param>
    /// <param name="fromLeft">A value received from the node to the left</param>
    /// <param name="fromBelow">A value received from the node below</param>
    /// <returns>An aggregator function should return a ValueType object computed as the aggregate of fromLeft and fromBelow</returns>
    public delegate Vtype Aggregator<KType, Vtype>(KType key, Vtype fromLeft, Vtype fromBelow);

    // This is used to "trick" C# into doing subtyping on my derived IAggregator objects
    internal interface IAggregateEventHandler
    {
        // In fact, three of the events return null, but the fourth fetches the marshalled aggregate state and returns byte[]
        byte[] AggEvent(int eventType, int vid, object key, object value, int offset);
        byte[] AggEvent(int eventType);
        string AggName();
        string AggState();
        object GetDValues();
        void LoadDValues(object fromBelow);
        void GotSGAggInfo(bool fromBelow, int level, int vid, object key, object value);
        Type GetKeyType();
        Type GetValueType();
    }

    /// <summary>
    /// Specifies how long a Query should wait for results and the action to take on timeout
    /// </summary>
    /// <remarks>
    /// Isis employs <bf>Timeout</bf> objects to specify desired actions when a Query has waited excessively long.
    /// These specify the delay until timeout (in milliseconds) and an action to take.
    /// </remarks>
    public class Timeout
    {
        internal int when;
        internal int action;
        internal string origin;
        /// <summary>
        /// <bf>TO_ABORTREPLY:</bf>  Abort this Query by simulating an <bf>AbortReply()</bf>
        /// </summary>
        public const int TO_ABORTREPLY = 0;
        /// <summary>
        /// <bf>TO_NULLREPLY:</bf>  Stop waiting for the slow group member by simulating a <bf>NullReply()</bf> from that member
        /// </summary>
        public const int TO_NULLREPLY = 1;
        /// <summary>
        /// <bf>TO_FAILURE:</bf>  Stop waiting for the slow group member by informing Isis that the member has failed.  This is an extreme action
        /// and will cause Isis to disconnect from that member and send it a poison pill, just in case it is still alive.  Use with care!
        /// </summary>
        public const int TO_FAILURE = 2;
        /// <summary>
        /// <bf>TO_AGGFAILURE:</bf>  For aggregations only, triggers an aggregation exception in the leader and, if the process responsible for the
        /// timeout can be identified, causes that process to be poisoned.
        /// </summary>
        public const int TO_AGGFAILURE = 4;

        /// <summary>
        /// Constructs a new member of the <bf>Timeout</bf> class
        /// </summary>
        /// <param name="to">Delay until timeout occurs, in ms</param>
        /// <param name="act">Action to take (for queries, one of TO_ABORTREPLY, TO_NULLREPLY, TO_FAILURE; for aggregation, TO_AGGFAILURE)</param>
        public Timeout(int to, int act) { when = to; action = act; }

        internal Timeout(int to, int act, string why) { when = to; action = act; origin = why; }
    }

    /// <summary>
    /// Various system constants.  Some accessible from outside users
    /// </summary>
    internal class Isis
    {
        internal static int ISIS_ORACLESIZE = 3;                // Target size for ORACLE;
        internal static int ISIS_INITVIAOOB = 50;               // If a group view has 50 or more members, use OOB transfer to initialize joining members.  Value should be larger than ORACLESIZE
        internal static bool ISIS_CANJOINORACLE = true;         // Can be overridden by environment variables
        internal static bool ISIS_OOBCLONEONSAMEHOST = false;   // If true, the OOB subsystem creates a new mapped file with a new name when the sender and receiver are on the same host (normally false, true only for debugging)
        internal const int ISIS_INDEXMEMBERS = 16;              // Maintain index if View members list is 16 or longer
        internal static int ISIS_MAXSYSTEMSIZE = 2048;          // Largest configurations that have been tested reasonably carefully.  We'll make this bigger and bigger over time.
        internal static int ISIS_ASYNCMTOTALLIMIT = 100;        // Inhibit new sends if more than this many messages are known to Isis.  Value adjusted each time view changes in ISISMEMBERS
        internal static int ISIS_MINASYNCMTOTAL = 50;           // MAXASYNCMTOTAL won't be allowed to become larger than this value
        internal static int ISIS_MAXRBACKLOG = ISIS_ASYNCMTOTALLIMIT * 2;
        internal static int ISIS_MAXDIRECTSENDS = 10;           // If a group has more than this many members, and a UDP-only multicast is attempted, Isis switches to an overlay multicast
        internal static long ISIS_MAXMSGLEN = 32 * 1024;        // Not used in TCP_ONLY mode.  Otherwise, we recommend keeping this fairly small to avoid excessive "memory pressure" on the kernel
        internal static long ISIS_OOBCHUNKSIZE;                 // Used in the out of band transfer logic, initialized there
        internal static long ISIS_MAXMSGLENTOTAL = 256 * (ISIS_MAXMSGLEN);  // Maximum size for data sent in an Send, OrderedSend or SafeSend, needed because Isis flow control can malfunction with extremely large objects
        internal static bool ISIS_MD5SIGS = true;               // If true, Isis uses MD5 signatures to sign every marshalled object, and won't demarshall (hence won't accept) unsigned messages
        internal static byte[] ISIS_AESKEY;                     // If provided, Isis encrypts the MD5 signatures with this key
        internal static int ISIS_ACKTHRESHOLD = 2;              // How many acks to wait for when doing COMMIT in the ORACLE
        internal static Aes ISIS_AES;
        internal static LockObject ISIS_AES_LOCK = new LockObject("ISIS_AES_LOCK");
        internal static Random ISIS_AESSEED;                    // Used to seed the initialization vector employed by AES
#if ___ANDROID__
        internal static bool ISIS_TCP_ONLY = true;             // If true, forces UNICAST_ONLY mode and sends all data over TCP point-to-point connection.  Requires ISIS_HOSTS
        internal static string ISIS_HOSTS = StaticItems.ipAddress;   // VN - the staticItems.ipaddress is the ip address entered in the Android App login screen
#else
        internal static bool ISIS_TCP_ONLY = false;             // If true, forces UNICAST_ONLY mode and sends all data over TCP point-to-point connection.  Requires ISIS_HOSTS
        internal static string ISIS_HOSTS = "";               // If non-empty, the names of nodes where ORACLE instances can be found (if any are running)
        //internal static string ISIS_HOSTS = "KenMacBookPro";    // If non-empty, the names of nodes where ORACLE instances can be found (if any are running)
        //internal static string ISIS_HOSTS = "KenT7500";       // If non-empty, the names of nodes where ORACLE instances can be found (if any are running)
#endif
        internal static bool ISIS_UNICAST_ONLY = false;         // If true, Isis uses no IPMC at all, even for startup.  (Value of MAXIPMCADDRS ignored in this case)
        internal static int ISIS_GROUPPORT = 11002;
        internal static int ISIS_MCRANGE_LOW = 5000;            // Isis allocates virtual IPMC addresses in this range; must be large enough to permit groups to have unique gaddrs.  
        internal static int ISIS_MCRANGE_HIGH = ISIS_MCRANGE_LOW + 500000;          // Physical ones use same range but are managed by Dr. Multicast
        internal static int ISIS_DEFAULT_PORTNOp = 9753;        // Listener port for incoming P2PSocket connection requests' AckSocket uses ISIS_DEFAULT_PORTNOp+1
        internal static int ISIS_DEFAULT_PORTNOa = ISIS_DEFAULT_PORTNOp + 1;        // Listener port for incoming AckSocket connection requests
        internal static int ISIS_MAXIPMCADDRS = 25;              // Limit on how many IPMC addresses can be in use other than for OOB code.  WARNING: Can be temporarily exceeded while remapping.
        internal static int OOBMAXIPMCADDRS = 250;               // Limit on how many IPMC addresses can be used for OOB code
        internal static byte[] ISIS_HDR = Msg.StringToBytes("->ISIS<-");
        internal static bool ISIS_LOG_CREATED = false;
        internal static Address foundOracle;
        internal static Address my_address;
        internal static byte[] my_address_bytes;
        internal static List<Address> recent_inquiries = new List<Address>();
        internal static LockObject recent_inquiries_lock = new LockObject("recent_inquiries_lock");
        internal static IPAddress[] ISIS_HOSTS_IPADDRS;         // Result of applying IPExtractAddrs to ISIS_HOSTS
        internal static Thread receiveThread;
        internal static int CLASSD = 224 << 24;
        internal static int ISIS_MSGPADDING = 0;                // Padding added to the end of a serialized message to hold the signature, if any
        internal static int nPhysAddrsInUse = 0;                // How many are being used right now
        internal static int ISIS_OVERHEAD = 746;                // How many bytes Isis adds as overhead (quite a few, in the worst case)
        internal static long ISIS_MUSTFRAGMENT;                 // Isis will fragment objects this large or larger...
        internal static long ISIS_FRAGLEN;                      // ... into objects of this size.  
        internal static int ISIS_MCMDBBSIZE = 512;
        internal static int ISIS_GCFREQ = 2000;                 // Forces garbage collection every 2 seconds, if more than a very few messages are "active" in Isis (currently, 10)
        internal static int ISIS_MCMDREPORTRATE = 5 * 60;       // Report MCMD stats every 5 minutes of runtime
        internal static long GRACEPERIOD = 20000;               // 20 second grace period during startup to deal with C# class loader locking I/O here and there
        internal static byte[] ISIS_OK = new byte[] { (byte)'O', (byte)'K' };
        internal static bool ISIS_SKIP_FIRSTINTERFACE = false;  // True if Isis shouldn't multicast on the first interface
        internal static bool ISIS_DONT_COMPRESS = false;
        internal static string ISIS_NETWORK_INTERFACES;
        internal static int[] InterfaceIds;
        internal static int ISIS_MIN2NDRTSEND = 2500;           // After retransmitting, minimum delay before doing it a second time (hence the 3rd send)
        internal static int ISIS_MAXRETRIES = 8;                // Much fudging needed to get these right
        internal static int ISIS_MAXLGRETRIES = 0;              // Much fudging needed to get these right.  My original theory was that this should be O(log(N)) but for now I've disabled it
        internal static int ISIS_DEFAULTTIMEOUT = 45000;        // 45 seconds
        internal static bool BigTimeouts = false;               // Set to true if ISIS_DEFAULTTIMEOUT has been doubled because the application is sending large objects
        internal static int ISIS_WARNAFTER = 5000;              // 5 seconds
        internal static int ISIS_REMAPDELAY = 15 * 60 * 1000;   // 15 minutes
        internal static int MS = 10000;                         // Ticks per millisecond (one tick is 100 nanoseconds)
        internal static int ISIS_ISBIG = 5;
        internal static int ISIS_TTL = 1;                       // 0: don't route; 1:LAN only.  Increase with care or you might DDoS the whole data center!
        internal static int ISIS_TOKEN_DELAY = 40;              // Pause between passing tokens in ms, 20x faster after recent activity.  Must be multiple of 20
        internal static bool WORKER_MODE = false;                // Set to true in RunAsWorker()
        internal static Address MY_MASTER = null;
        internal static int OneToOneCntr = 0;                   // For message id's on one-to-one ping messages, which show up with gaddr=NULLADDRESS
        internal static bool ISIS_UDPCHKSUM = false;
        internal static int my_pid = Process.GetCurrentProcess().Id;
        internal static IPAddress my_IPaddress;             // Address of this process
        internal static string my_host;
        internal static bool IAmOracle = false;
        internal static Group ORACLE;
        internal static Group ISISMEMBERS;
        internal static Thread OracleViewThread;                 // Only active in the Oracle group
        internal static ViewDelta[] Proposed;
        internal static int LeaderId;
        volatile internal static bool RunningLeaderLogic;                 // True once this instance is running the "oracle leader" code
        internal static Address NULLADDRESS = new Address();
        internal static Address ClientOf;
        internal static long OracleFailedAt;
        internal static int newClientOfCnt;
        internal static bool ISIS_LOGGED = true;                 // False disables all logging by Isis
        internal static bool ISIS_LARGE = false;                 // True if you want Isis to configure ISISMEMBERS as a "large" group
        internal static bool ISIS_MUTE = false;
        internal static bool ISIS_IGNOREPARTITIONS = true;
        internal static string ISIS_LOGDIR = "c:\\logs";
        internal static Stream my_logstream;
        internal static Semaphore lSema = new Semaphore(1, Int32.MaxValue);
        internal static int MapperEpochId;
        internal static string ISIS_NETMASK = "";
        internal static string ISIS_SUBNET = "";
        internal static bool ISIS_GRACEFULSHUTDOWN = false;

        internal static List<Address> RIPList = new List<Address>();
        internal static LockObject RIPLock = new LockObject("RIPLock");

        internal static string RIPListState()
        {
            string s;
            using (new LockAndElevate(RIPLock))
                s = "RIPList = " + Address.VectorToString(RIPList.ToArray());
            using (new LockAndElevate(Group.GroupRIPLock))
                s += "; Group RIPList = " + Address.VectorToString(Group.GroupRIPList.ToArray()) + "\r\n";
            return s;
        }

        internal const int JOIN = -1;
        internal const int LEAVE = -2;
        internal const int FDETECTION = -3;
        internal const int PROPOSE = -4;
        internal const int INITIALVIEW = -5;
        internal const int INQUIRE = -6;
        internal const int COMMIT = -7;
        internal const int FANNOUNCE = -8;
        internal const int ORDEREDSEND = -9;
        internal const int SETORDER = -10;
        internal const int RELAYJOIN = -11;
        internal const int RELAYLEAVE = -12;
        internal const int STATEXFER = -13;
        internal const int TERMINATE = -14;
        internal const int RELAYTERM = -15;
        internal const int REMAP = -16;
        internal const int SAFESEND = -17;
        internal const int SAFEDELIVER = -18;
        internal const int JOINFAILED = -19;
        internal const int RELAYSEND = -20;
        internal const int DALDONE = -21;
        internal const int ISSTABLE = -22;
        internal const int BECLIENT = -23;
        internal const int ORACLERUNNING = -24;
        internal const int PARTITIONED = -25;
        internal const int RELAYREGISTERVG = -26;
        internal const int REGISTERVG = -27;
        internal const int IM_DHT_PUT = -28;
        internal const int IM_DHT_GET = -29;
        internal const int IM_UDP_TUNNEL = -30;
        internal const int IM_IPMC_TUNNEL = -31;
        internal const int IM_IPMC_VIEWS = -32;
        internal const int CRYPTOWRAPPED = -33;
        internal const int CLIENTWRAPPED = -34;
        internal const int FRAGMENT = -35;
        internal const int CAUSALSEND = -36;
        internal const int DISKLOGGER = -37;
        internal const int PING = -38;
        internal const int CANBEORACLE = -39;
        internal const int LOCKREQ = -40;
        internal const int SGAGGREGATE = -41;
        internal const int OUTOFBAND = -42;
        internal const int SYSTEMREQS = 42;

        internal static string rToString(object o)
        {
            string[] names = new string[] { "JOIN", "LEAVE", "FDETECTION", "PROPOSE", "INITIALVIEW", "INQUIRE", "COMMIT", "FANNOUNCE", "ORDEREDSEND",
                "SETORDER", "RELAYJOIN", "RELAYLEAVE", "STATEXFER", "TERMINATE", "RELAYTERM", "REMAP", "SAFESEND", "SAFEDELIVER", "JOINFAILED",
                "RELAYSEND", "DALDONE", "ISSTABLE", "BECLIENT", "ORACLERUNNING", "PARTITIONED", "RelayRegisterVG", "RegisterVG",
                "DHTPut", "DHTGet", "UDP Tunnel", "IPMC Tunnel", "IPMC Views", "CRYPTOWRAP", "CLIENT-TO-GROUP", "FRAGMENT", "CAUSAL SEND", "DISK LOGGER", "PING", 
                "CAN BE ORACLE", "LOCKREQ", "Small Group AGGREGATE", "OutOfBand"
            };
            if (o.GetType().Equals(typeof(int)))
            {
                int r = (-(int)o) - 1;
                if (r >= 0 && r < names.Length)
                    return names[r];
                return "Hander[" + (int)o + "]";
            }
            return o.GetType().ToString();
        }

        public static void WriteLine()
        {
            WriteLine("");
        }

        public static void WriteLine(string what)
        {
            if (!IsisSystem.IsisActive && IsisSystem.IsisWasActive)
                return;
            if (ISIS_LOGGED)
                Write(what + "\r\n");
        }

        //  This method is unsafe and should only be used internally in Isis.cs for debugging purposes.
        internal static string ExtractStackTrace()
        {
            return ExtractStackTrace(Thread.CurrentThread, int.MaxValue);
        }

        //  This method is unsafe and should only be used internally in Isis.cs for debugging purposes.
        internal static string ExtractStackTrace(Thread targetThread, int depth)
        {
#if EXTRACTCALLSTACKS
            try
            {
                if (targetThread != Thread.CurrentThread && targetThread.ThreadState == System.Threading.ThreadState.Running)
                    return "(can't extract stack trace on a running thread)";
                string st = "{ ";
                if (Thread.CurrentThread.Name != null)
                    st = "Thread[" + Thread.CurrentThread.Name + "]{ ";
                StackTrace stackTrace = new StackTrace(targetThread, false);
                StackFrame[] stackFrames = stackTrace.GetFrames();
                // write call stack method names
                int ignore = 2;
                foreach (StackFrame stackFrame in stackFrames)
                    if (ignore-- <= 0 && depth-- >= 0)
                        st += stackFrame.GetMethod().Name + "... ";
                return st + "}";
            }
            catch (ThreadStateException)
            {
                return "(attempt to trace thread triggered thread state exception)";
            }
#else // EXTRACTCALLSTACKS
            return "??";
#endif // EXTRACTCALLSTACKS
        }

        public static void Write(string what)
        {
            if (ISIS_MUTE == false)
                Console.Write(what);
            lSema.WaitOne();
            if (ISIS_LOGGED && my_logstream != null)
            {
                byte[] buf = Msg.StringToBytes(what);
                my_logstream.Write(buf, 0, buf.Length);
                my_logstream.Flush();
            }
            lSema.Release(1);
        }

        public static void CloseLog()
        {
            lSema.WaitOne();
            if (ISIS_LOGGED)
            {
                ISIS_LOGGED = false;
                my_logstream.Flush();
                Isis.my_logstream.Close();
                Isis.my_logstream = null;
            }
            lSema.Release(1);
        }

        internal static void ArrayResize<T>(ref T[] oldArray, int len)
        {
            Array.Resize<T>(ref oldArray, len);
        }

        internal class TCB
        {
            internal int id;
            internal long when;
            internal TimerCallback cb;

            internal TCB(int tid, long t, TimerCallback tcb)
            {
                id = tid;
                when = t;
                cb = tcb;
            }
        }

        private static LinkedList<TCB> timer_list = new LinkedList<TCB>();
        private static LockObject timer_lock = new LockObject("timer_lock");
        private static int ENDOFTIME = timer_list.AddLast(new TCB(-1, Int64.MaxValue, new TimerCallback(delegate() { throw new Exception("end of time"); }))).Value.id;
        private static long STARTEDAT = DateTime.Now.Ticks;
        private static Semaphore timer_wait = new Semaphore(0, int.MaxValue);
        internal static Thread timer_thread;

        internal static long NOW()
        {
            return DateTime.Now.Ticks / 10000 - STARTEDAT / 10000;
        }

        internal static long NOWus()
        {
            return DateTime.Now.Ticks / 10 - STARTEDAT / 10;
        }

        internal static long CurTOD()
        {
            return DateTime.Now.Ticks;
        }

        internal static int TID;

        /// <summary>
        /// Register for a callback from Isis after a designated number of ms, using a new thread.
        /// </summary>
        /// <param name="ms">Delay in milliseconds</param>
        /// <param name="cb">Method to call, on a new thread</param>
        /// <returns></returns>
        public static int OnTimerThread(int ms, TimerCallback cb)
        {
            return OnTimer(ms, delegate() { new Thread(delegate() { try { Thread.CurrentThread.Name = "OnTimer callback: " + cb.Method; cb(); } catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); } IsisSystem.ThreadTerminationMagic(); }).Start(); });
        }

        /// <summary>
        /// Register for a callback from Isis after a designated number of ms
        /// </summary>
        /// <param name="ms">Delay in milliseconds</param>
        /// <param name="cb">Method to call</param>
        /// <returns></returns>
        public static int OnTimer(long ms, TimerCallback cb)
        {
            ms += NOW();
            TCB newtcb = new TCB(++TID, ms, cb);
            return InsertOnTimerQueue(newtcb);
        }

        private static int InsertOnTimerQueue(TCB newtcb)
        {
            using (new LockAndElevate(timer_lock))
            {
                bool new_first = true;
                for (LinkedListNode<TCB> tcbnode = timer_list.First; tcbnode != null; tcbnode = tcbnode.Next)
                    if (tcbnode.Value.when > newtcb.when)
                    {
                        timer_list.AddBefore(tcbnode, newtcb);
                        if ((IsisSystem.Debug & IsisSystem.TIMERS) != 0)
                            IsisSystem.WriteLine("InsertOnTimerQueue add callback with id " + newtcb.id + " registered before id " + tcbnode.Value.id + " for time " + newtcb.when + " at time " + NOW());
                        timer_wait.Release(1);
                        return TID;
                    }
                    else
                        new_first = false;
                timer_list.AddLast(newtcb);
                if ((IsisSystem.Debug & IsisSystem.TIMERS) != 0)
                    IsisSystem.WriteLine("InsertOnTimerQueue add callback with id " + newtcb.id + " registered at tail for time " + newtcb.when + " at time " + NOW());
                if (new_first)
                    timer_wait.Release(1);
                return TID;
            }
        }

        /// <summary>
        /// Adjust a callback to occur at a new deadline time
        /// </summary>
        /// <param name="tid">Timer ID</param>
        /// <param name="newDeadline">new deadline, in ms relative to NOW</param>
        /// <returns></returns>
        public static void TimerReset(int tid, long newDeadline)
        {
            if (tid == ENDOFTIME)
                return;
            LinkedListNode<TCB> tcbnode = null;
            newDeadline += NOW();
            using (new LockAndElevate(timer_lock))
            {
                for (tcbnode = timer_list.First; tcbnode != null; tcbnode = tcbnode.Next)
                    if (tcbnode.Value.id == tid)
                    {
                        timer_list.Remove(tcbnode);
                        break;
                    }
            }
            if (tcbnode != null)
            {
                tcbnode.Value.when = newDeadline;
                InsertOnTimerQueue(tcbnode.Value);
            }
        }

        public static void TimerCancel(int tid)
        {
            if (tid == ENDOFTIME)
                return;
            using (new LockAndElevate(timer_lock))
            {
                for (LinkedListNode<TCB> tcb = timer_list.First; tcb != null; tcb = tcb.Next)
                    if (tcb.Value.id == tid)
                    {
                        timer_list.Remove(tcb);
                        break;
                    }
            }
        }

        internal static void TimerThread()
        {
            while (!IsisSystem.IsisActive)
                Thread.Sleep(250);
            try
            {
                while (IsisSystem.IsisActive)
                {
                    IsisSystem.RTS.ThreadCntrs[2]++;
                    LinkedListNode<TCB> tcb;
                    List<TimerCallback> cbs = new List<TimerCallback>();
                    IsisSystem.CheckParentThread();
                    using (new LockAndElevate(timer_lock))
                    {
                        while ((tcb = timer_list.First).Value.when <= NOW())
                        {
                            if ((IsisSystem.Debug & IsisSystem.TIMERS) != 0)
                                IsisSystem.WriteLine("TimerThread to call Timer Callback with id " + tcb.Value.id + " registered for time " + tcb.Value.when + " at time " + NOW());
                            timer_list.Remove(tcb);
                            cbs.Add(tcb.Value.cb);
                        }
                    }
                    foreach (TimerCallback cb in cbs)
                        cb();
                    int delay;
                    using (new LockAndElevate(timer_lock))
                        delay = (int)Math.Min(2500, timer_list.First.Value.when - NOW());
                    if (delay > 0)
                        timer_wait.WaitOne(delay);
                }
            }
            catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
        }

        internal static string GetTimerState()
        {
            string s = "Timer State: NOW = " + TimeToString(NOW()) + "... ";
            //Omit unless explictly requested: too verbose...
            if ((IsisSystem.Debug & IsisSystem.TIMERS) != 0)
                using (new LockAndElevate(timer_lock))
                {
                    for (LinkedListNode<TCB> tcb = timer_list.First; tcb != null; tcb = tcb.Next)
                        if (tcb.Value.when != Int64.MaxValue)
                            s += "\r\n     [" + tcb.Value.id + "] At " + TimeToString(tcb.Value.when) + " do callback to " + tcb.Value.cb.Method;
                }
            s += "\r\n" + ReliableSender.HeardFromState();
            return s;
        }

        internal static string TimeToString(long when)
        {
            string time = when.ToString();
            if (time.Length < 4)
                time = "0000".Substring(time.Length) + time;
            time = time.Substring(0, time.Length - 3) + "." + time.Substring(time.Length - 3);
            return time;
        }

        public static void NodeHasFailed(Address which, string howDiscovered, bool inhibitReport)
        {
            if (Isis.ClientOf != null && Isis.ClientOf.Equals(which))
                if (Isis.ISIS_ORACLESIZE == 1)
                    throw new IsisShutdown("System shutdown: Isis was configured with ISIS_ORACLESIZE=1 and the Oracle has failed or terminated.");
                else if (Isis.OracleFailedAt != 0 && (Isis.NOW() - Isis.OracleFailedAt) > Isis.ISIS_DEFAULTTIMEOUT)
                    throw new IsisShutdown("System shutdown: After Isis ORACLE member " + Isis.ClientOf + " failed, this client was unable to contact any other ORACLE member.");
                else if (Isis.OracleFailedAt == 0)
                    Isis.OracleFailedAt = Isis.NOW();
            if (!ORACLE.HasFirstView || !IsisSystem.IsisActive || IsisSystem.IsisRestarting)
                return;
            using (new LockAndElevate(Isis.RIPLock))
                if (Isis.RIPList.Contains(which) == false)
                    Isis.RIPList.Add(which);
                else
                    return;
            // Clean up the CanBeOracleList to make sure we don't try to add a deceased process to the ORACLE
            using (new LockAndElevate(CanBeOracleListLock))
                CanBeOracleList.Remove(which);
            string name = Thread.CurrentThread.Name ?? "Unnamed thread";
            if (which.isMyAddress())
                throw new IsisException("[" + name + "] Isis node " + Isis.my_address + " detected its own failure(" + howDiscovered + "); shutting down: " + IsisSystem.GetState());
            if ((IsisSystem.Debug & IsisSystem.FAILURES) != 0)
                Isis.WriteLine("[" + name + "] NodeHasFailed " + howDiscovered + " failure discovery event for " + which + ", " + IsisSystem.GetState());
            ReliableSender.NodeHasFailed(which);
            if (!inhibitReport)
            {
                int rank = ORACLE.theView.GetRankOf(which);
                if (rank != -1)
                    ORACLE.theView.noteFailed(rank);
                if (Isis.ClientOf == null)
                    ORACLE.doSend(false, false, Isis.FDETECTION, which);
                else if (Isis.ClientOf.Equals(which) == false)
                    ORACLE.doP2PSend(Isis.ClientOf, true, Isis.FDETECTION, which);
                else
                {
                    View v = Isis.ISISMEMBERS.theView;
                    if (v != null)
                    {
                        foreach (Address m in v.members)
                        {
                            if (m.isMyAddress())
                                continue;
                            using (new LockAndElevate(Isis.RIPLock))
                                if (Isis.RIPList.Contains(m))
                                    continue;
                            if ((IsisSystem.Debug & IsisSystem.FAILURES) != 0)
                                Isis.WriteLine("[" + name + "] Sending FDETECTION notification in Isis.ISISMEMBERS to " + m);
                            Isis.ISISMEMBERS.doP2PSend(m, true, Isis.FDETECTION, which);
                        }
                    }
                    // We need to give the new ORACLE leader time to take over and get back in touch
                    // This loop will run for 30 seconds, checking 4 times per second
                    for (int nt = 0; nt < 120; nt++)
                    {
                        Thread.Sleep(250);
                        if (ORACLE.theView.GetRankOf(which) == -1)
                            return;
                    }
                    IsisSystem.Shutdown("Client lost connectivity to core Isis system");
                }
            }
        }

        public static void GroupHasFailed(string why)
        {
            throw new IsisException(why);
        }

        internal static bool BVCompare(byte[] a, byte[] b)
        {
            if (a.Length != b.Length)
                return false;
            for (int i = 0; i < a.Length; i++)
                if (a[i] != b[i])
                    return false;
            return true;
        }

        internal static long GetTime()
        {
            DateTime Now = DateTime.Now;
            return Now.Ticks;
        }

        private static bool rentry = false;

        internal static IPAddress setMyAddress()
        {
            if (rentry)
                return Isis.my_IPaddress;
            rentry = true;
            try
            {
                IDictionary environmentVariables = Environment.GetEnvironmentVariables();
                foreach (DictionaryEntry de in environmentVariables)
                {
                    if (de.Key.Equals("ISIS_PORTNO"))
                        ISIS_GROUPPORT = int.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_SKIP_FIRSTINTERFACE"))
                        ISIS_SKIP_FIRSTINTERFACE = bool.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_DONT_COMPRESS"))
                        ISIS_DONT_COMPRESS = bool.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_LOGGED"))
                        ISIS_LOGGED = bool.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_LARGE"))
                        ISIS_LARGE = bool.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_MUTE"))
                        ISIS_MUTE = bool.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_IGNOREPARTITIONS"))
                        ISIS_IGNOREPARTITIONS = bool.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_GCFREQ"))
                        ISIS_GCFREQ = int.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_MCRANGE_LOW"))
                        ISIS_MCRANGE_LOW = int.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_MCRANGE_HIGH"))
                        ISIS_MCRANGE_HIGH = int.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_MAXIPMCADDRS"))
                        ISIS_MAXIPMCADDRS = int.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_MAXDIRECTSENDS"))
                        ISIS_MAXDIRECTSENDS = int.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_UNICAST_ONLY"))
                        ISIS_UNICAST_ONLY = bool.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_TCP_ONLY"))
                        ISIS_TCP_ONLY = bool.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_UDPCHKSUM"))
                        ISIS_UDPCHKSUM = bool.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_CANJOINORACLE"))
                        ISIS_CANJOINORACLE = bool.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_MCMDREPORTRATE"))
                        ISIS_MCMDREPORTRATE = int.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_TTL"))
                        ISIS_TTL = int.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_TOKEN_DELAY"))
                        ISIS_TOKEN_DELAY = Math.Min(1000, Math.Max(1, int.Parse((string)de.Value)));
                    else if (de.Key.Equals("ISIS_LOGDIR"))
                        ISIS_LOGDIR = (string)de.Value;
                    else if (de.Key.Equals("ISIS_NETWORK_INTERFACES"))
                        ISIS_NETWORK_INTERFACES = (string)de.Value;
                    else if (de.Key.Equals("ISIS_HOSTS"))
                        ISIS_HOSTS = (string)de.Value;
                    else if (de.Key.Equals("ISIS_NETMASK"))
                        ISIS_NETMASK = (string)de.Value;
                    else if (de.Key.Equals("ISIS_SUBNET"))
                        ISIS_SUBNET = (string)de.Value;
                    else if (de.Key.Equals("ISIS_GRACEFULSHUTDOWN"))
                        ISIS_GRACEFULSHUTDOWN = bool.Parse((string)de.Value);
                    else if (de.Key.Equals("ISIS_PORTNOp"))
                    {
                        ISIS_DEFAULT_PORTNOp = int.Parse((string)de.Value);
                        ISIS_DEFAULT_PORTNOa = ISIS_DEFAULT_PORTNOp + 1;
                    }
                    else if (de.Key.Equals("ISIS_PORTNOa"))
                        throw new IsisException("ISIS_PORTNOa cannot be directly changed");
                    else if (de.Key.Equals("ISIS_AESKEY"))
                    {
                        Group.doInitializeAes(out ISIS_AES);
                        ISIS_AESKEY = byteVecParse((string)de.Value);
                        ISIS_MD5SIGS = true;
                    }
                    else if (de.Key.Equals("COMPUTERNAME") || de.Key.Equals("HOSTNAME"))
                        Isis.my_host = (string)de.Value;
                }

                if (ISIS_AES != null)
                    ISIS_AESSEED = new Random();
                if (ISIS_MD5SIGS)
                {
                    if (ISIS_AES == null)
                        ISIS_MSGPADDING = 16;    // Length of MD5 hash
                    else
                        ISIS_MSGPADDING = 48;    // Length of encrypted MD5 hash
                }
                ISIS_TOKEN_DELAY = Math.Max(20, (ISIS_TOKEN_DELAY / 20) * 20);
                ISIS_HOSTS_IPADDRS = ExtractHostIPAddrs(ISIS_HOSTS);

                if (ISIS_TCP_ONLY)
                {
                    ISIS_DEFAULTTIMEOUT *= 3;
                    ISIS_MAXMSGLEN = int.MaxValue;
                }

#if __MonoCS__
                Thread.Sleep(my_pid % 1000);  // Hack to avoid triggering some sort of Dns bug, observed only on Mono
#endif

                if (ISIS_UNICAST_ONLY == false)
                {
                    string[] which = null;
                    if (ISIS_NETWORK_INTERFACES != null)
                    {
                        string s = ISIS_NETWORK_INTERFACES;
                        which = new string[50];
                        int n = 0, idx = 0;
                        while (idx < s.Length && n < which.Length)
                        {
                            int from = idx;
                            while (idx != s.Length && s[idx] != ',')
                                ++idx;
                            which[n++] = s.Substring(from, idx++ - from);
                        }
                        Isis.ArrayResize<string>(ref which, n);
                    }
                    //VN - change on Android - getNetworkInterfaces() does not execute on Mono for android.
#if ___ANDROID__
                    InterfaceIds = new int[0];
#else
                    InterfaceIds = ReliableSender.getNetworkInterfaces(which);
#endif

                    if ((IsisSystem.Debug & IsisSystem.INTERFACES) != 0)
                    {
                        Isis.Write("ISIS_NETWORK_INTERFACES = <");
                        int cnt = 0;
                        foreach (int id in InterfaceIds)
                        {
                            if (cnt++ != 0)
                                Isis.Write(" ");
                            if (ISIS_SKIP_FIRSTINTERFACE)
                                Isis.Write("SKIPPING[" + id.ToString() + "]");
                            else
                                Isis.Write(id.ToString());
                        }
                        Isis.WriteLine(">");
                    }
                    if (ISIS_SKIP_FIRSTINTERFACE)
                    {
                        for (int i = 0; i < InterfaceIds.Length - 1; i++)
                            InterfaceIds[i] = InterfaceIds[i + 1];
                        Isis.ArrayResize<int>(ref InterfaceIds, InterfaceIds.Length - 1);
                    }
                }
                else
                    ReliableSender.getNetworkInterfaces(null);

                if (Isis.my_IPaddress == null)
                {
                    Isis.WriteLine("WARNING: No network interface supports IPv4... using `localhost=127.0.0.1' for my IP address (network connectivity will be limited)");
                    Isis.my_IPaddress = new IPAddress(new byte[4] { 127, 0, 0, 1 });
                }

                //VN - change on Android - storing the log file in the external storage directory in the Android device.
                if (ISIS_LOGGED && ISIS_LOG_CREATED == false)
                {
#if __MonoCS__
                    string directory = "./OUT";
                    string fname = "./OUT/ISIS-" + Isis.my_IPaddress.ToString() + "-" + my_pid + ".log";
#elif ___ANDROID__ // !__MonoCS__ but ___ANDROID__
                    string directory = Path.Combine(Android.OS.Environment.ExternalStorageDirectory.ToString(), ISIS_LOGDIR);
                    string fname = Path.Combine(directory, "ISIS-" + my_pid + ".log");
#else
                    string directory = ISIS_LOGDIR;
                    string fname = Path.Combine(ISIS_LOGDIR, "ISIS-" + my_pid + ".log");
#endif // !__MonoCS__
                    try
                    {
                        System.IO.Directory.CreateDirectory(directory);
                    }
                    catch { }
                    try
                    {
                        File.Delete(fname);
                    }
                    catch { }
                    try
                    {
#if !___ANDROID__
                        my_logstream = new FileStream(fname, FileMode.CreateNew);
#else
                        my_logstream = new FileStream(fname, FileMode.Create, FileAccess.Write, FileShare.Write);//VN - the file should have wrtie access
#endif

                        ISIS_LOG_CREATED = true;
                        string rev = "$Rev: 1306 $";
                        rev = rev.Substring(rev.IndexOf(" ") + 1);
                        rev = "Isis V2.0 Revision " + rev.Substring(0, rev.IndexOf(" "));
                        string settings = "<" + Isis.my_host + ": " + Isis.my_IPaddress + ">\r\nMajor Isis Runtime Settings: ORACLESIZE = " + ISIS_ORACLESIZE + "; ";
                        if (ISIS_TCP_ONLY)
                            settings += "TCP_ONLY; ";
                        else if (ISIS_UNICAST_ONLY)
                            settings += "UNICAST_ONLY; ";
                        if (ISIS_HOSTS.Length > 0)
                            settings += "ISIS_HOSTS = {" + ISIS_HOSTS + "}; ";
                        settings += "\r\nP2P/ACK port numbers = {" + ISIS_DEFAULT_PORTNOp + "/" + ISIS_DEFAULT_PORTNOa + "}; ";
                        if (!ISIS_TCP_ONLY && !ISIS_UNICAST_ONLY)
                            settings += "IPMC portno = " + ISIS_GROUPPORT +
                                "\r\nIPMC address range = {" + MCMDSocket.PMCAddr(CLASSD + ISIS_MCRANGE_LOW) + "-" + MCMDSocket.PMCAddr(CLASSD + ISIS_MCRANGE_HIGH) +
                                "}, MAXIPMCADDRS in use = " + ISIS_MAXIPMCADDRS;
#if __MonoCS__
                        settings = "; Compiled for MONO; " + settings;
#endif
#if __ANDROID__
                        settings = "; Compiled for Android; " + settings;
#endif
                        byte[] header = Msg.StringToBytes(rev + ": " + DateTime.Now.ToShortDateString() + " " + DateTime.Now.ToLongTimeString() +
                            settings + "\r\n---------------------------------------------------------------------------------------------\r\n");
                        my_logstream.Write(header, 0, header.Length);
                    }
                    catch (IOException e)
                    {
                        ISIS_LOGGED = false;
                        my_logstream = null;
                        Console.WriteLine("WARNING: Unable to create Isis log file: " + fname + "(" + e + ")");
                    }
                }

                if (ISIS_TCP_ONLY)
                    ISIS_UNICAST_ONLY = false;

                if ((ISIS_UNICAST_ONLY || ISIS_TCP_ONLY) && ISIS_HOSTS_IPADDRS == null)
                    throw new IsisException("ISIS: UNICAST_ONLY or TCP_ONLY mode but you didn't specify ISIS_HOSTS for initial rendezvous!");
                ISIS_FRAGLEN = (Isis.ISIS_MAXMSGLEN - Isis.ISIS_OVERHEAD * 2);
                ISIS_MUSTFRAGMENT = (Isis.ISIS_MAXMSGLEN - Isis.ISIS_OVERHEAD);
                Msg.Initialize();
                my_address = new Address(Isis.my_IPaddress, my_pid);
                ReliableSender.Init();
                my_address_bytes = Msg.toBArray(my_address);
                Msg.doRegisterType(typeof(Isis.ViewDelta), Msg.VIEWDELTA);
                Msg.doRegisterType(typeof(Isis.UnstableList), Msg.UNSTABLE);
                Msg.doRegisterType(typeof(Group.tokenInfo), Msg.TOKENINFO);
                Msg.doRegisterType(typeof(Group.FlushAggKey), Msg.FLSHAGGKEY);
                Msg.doRegisterType(typeof(MCMDSocket.GRPair), Msg.GRPAIR);
                Msg.doRegisterType(typeof(ReliableSender.TCPhdr), Msg.TCPHDR);
                Msg.doRegisterType(typeof(Group.LockInfo), Msg.LOCKINFO);
                Msg.doRegisterType(typeof(QueryKey<>), Msg.QUERYKEY);
                Msg.doRegisterType(typeof(Group.DHTItem), Msg.DHTITEM);
                Msg.doRegisterType(typeof(Group.osspq), Msg.OSSPQ);
                Msg.doRegisterType(typeof(Group.OOBRepInfo), Msg.OOBREPINFO);
                ISIS_OOBCHUNKSIZE = ISIS_MAXMSGLEN - 2048;

                Thread t = new Thread(ReliableSender.TokenThread);
                t.Name = "Isis token-loop thread";
                t.Start();
                /*
                 * The rather tortured logic that follows tries to deal with a wide range of startup scenarios including multicast supported or not,
                 * Oracle already running or not, several copies launched simultaneously as opposed to one by one, machines lightly or heavily loaded, etc.
                 * All of this makes for a "mess".  Sorry.
                 */
                ORACLE = new Group("ORACLE");
                SetupORACLE();
                if (Isis.WORKER_MODE)
                    return null;
                int OracleTries = 0;
            again:
                if (OracleTries > 0)
                    Thread.Sleep(OracleTries * 5000);
                bool cantBeOracle = false;
                int theDelay = 1500;
                if (!ISIS_CANJOINORACLE || (Isis.ISIS_TCP_ONLY && (Isis.my_address.p2pPort != Isis.ISIS_DEFAULT_PORTNOp || Isis.my_address.ackPort != Isis.ISIS_DEFAULT_PORTNOa)))
                {
                    cantBeOracle = true;
                    ISIS_CANJOINORACLE = false;
                    Isis.WriteLine("Isis: This instance can't be the ORACLE, joining as a client");
                }
                for (int retry = 0; Isis.ClientOf == null && !IsisSystem.fastStart && retry < (cantBeOracle ? 20 : 3) && Isis.foundOracle == null; retry++)
                {
                    tryToJoin();
                    if (!cantBeOracle && Isis.foundOracle == null && retry == 0)
                        Isis.WriteLine("Isis: Searching for the Isis ORACLE...");
                    Thread.Sleep(theDelay);
                    if (Isis.ClientOf != null || (ORACLE.theView != null && Isis.ORACLE.theView.GetMyRank() != -1))
                        break;
                    Thread.Sleep(theDelay);
                }
                if (Isis.foundOracle != null && Isis.ClientOf == null)
                {
                    Isis.Write("Isis: Found the Isis.ORACLE service, attempting to connect.");
                    for (int retry = 0; (!ORACLE.HasFirstView || ORACLE.theView.GetMyRank() == -1) && Isis.ClientOf == null && retry < 10; retry++)
                    {
                        if (Isis.ClientOf == null)
                            tryToJoin();
                        theDelay = Math.Min(6000, theDelay);
                        if ((IsisSystem.Debug & IsisSystem.STARTSEQ) != 0)
                            Isis.WriteLine("WARNING: Found the ORACLE but it was still restarting... delaying " + theDelay + "ms before retrying join request... " + retry);
                        Thread.Sleep(theDelay);
                        Isis.Write(".");
                        theDelay += 500;
                    }
                    Isis.WriteLine("");
                    if (Isis.ClientOf == null && (!ORACLE.HasFirstView || Isis.ORACLE.theView.GetMyRank() == -1))
                        throw new IsisException("Isis ORACLE is " + Isis.foundOracle + " but attempt to connect with it failed");
                }
                if (cantBeOracle && Isis.ClientOf == null)
                {
                    if (++OracleTries < 2)
                        goto again;
                    else if (Isis.foundOracle == null)
                        throw new IsisException("I can't be the ORACLE but was unable to contact the ORACLE in ISIS_TCP_ONLY / ISIS_UNICAST_ONLY mode");
                    else
                        throw new IsisException("ORACLE is " + Isis.foundOracle + " but I was unable to connect with it in ISIS_TCP_ONLY / ISIS_UNICAST_ONLY mode");
                }
                Thread hb = new Thread(Group.GroupMemberHeartBeat);
                hb.Name = "Isis All-Groups HeartBeat thread";
                hb.Start();
                if (Isis.ClientOf == null)
                {
                    hb = new Thread(Isis.ORACLE.OracleHeartBeat);
                    hb.Name = "Isis <ORACLE> HeartBeat thread";
                    hb.Start();
                    if (ORACLE.HasFirstView == false)
                    {
                        if ((IsisSystem.Debug & IsisSystem.STARTSEQ) != 0)
                            Isis.WriteLine("Restarting the ORACLE in " + IsisSystem.GetState());
                        List<Address> toNotify = new List<Address>();
                        using (new LockAndElevate(ReliableSender.TCPLock))
                            foreach (ReliableSender.TCPSockInfo tcpsi in ReliableSender.TCPList)
                                if (!toNotify.Contains(tcpsi.dest))
                                    toNotify.Add(tcpsi.dest);
                        using (new LockAndElevate(ReliableSender.ConnectingLock))
                            foreach (ReliableSender.AWPair awp in ReliableSender.ConnectingTo)
                                if (!toNotify.Contains(awp.dest))
                                    toNotify.Add(awp.dest);
                        foreach (Address who in toNotify)
                            Isis.ORACLE.doPureP2PSend(who, true, Isis.JOIN, Isis.my_address);
                    }
                }
                return Isis.my_IPaddress;
            }
            catch (Exception e)
            {
                Isis.WriteLine("IsisLib: Initialization error <" + e + ">");
                IsisSystem.IsisActive = false;
            }
            return null;
        }

        private static string hex = "0123456789ABCDEF";

        private static byte[] byteVecParse(string arg)
        {
            int idx = 0;
            if (arg.Length != (ISIS_AES.KeySize * 2))
                throw new IsisException("ISIS_AESKEY: argument has incorrect length (should be a " + (ISIS_AES.KeySize) + "-byte/" + (ISIS_AES.KeySize * 8) + "-bit vector, encoded as a hexstring");
            byte[] bvec = new byte[ISIS_AES.KeySize];
            for (int off = 0; off < bvec.Length; off++)
            {
                char c1 = arg[idx++];
                char c2 = arg[idx++];
                bvec[off] = (byte)((hex.IndexOf(c1) << 8) | hex.IndexOf(c2));
            }
            return bvec;
        }

        private static IPAddress[] ExtractHostIPAddrs(string hlist)
        {
            if (hlist == null || hlist.Length == 0)
                return null;

            int nContacts = 0;
            hlist += ",";
            for (int i = 0; i < hlist.Length; i++)
                if (hlist[i] == ',')
                    nContacts++;
            IPAddress[] newISIS_HOSTS = new IPAddress[nContacts];
            nContacts = 0;
            for (int i = 0; i < hlist.Length; i++)
                if (hlist[i] == ',')
                {
                    newISIS_HOSTS[nContacts++] = LastIPv4(hlist.Substring(0, i));
                    hlist = hlist.Substring(i + 1);
                    i = 0;
                }
            return newISIS_HOSTS;
        }

        internal static bool IPv4InNetwork(IPAddress address, IPAddress netmask, IPAddress subnet)
        {
            if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork) // IPv4
            {
                Byte[] addressOctets = address.GetAddressBytes();
                Byte[] netmastOctets = netmask.GetAddressBytes();
                Byte[] subnetOctets = subnet.GetAddressBytes();
                return
                    (subnetOctets[0] == (addressOctets[0] & netmastOctets[0])) &&
                    (subnetOctets[1] == (addressOctets[1] & netmastOctets[1])) &&
                    (subnetOctets[2] == (addressOctets[2] & netmastOctets[2])) &&
                    (subnetOctets[3] == (addressOctets[3] & netmastOctets[3]));
            }
            else // IPv6
            {
                return false;
            }
        }

        private static Dictionary<string, IPAddress> IPv4Map = new Dictionary<string, IPAddress>();
        private static LockObject IPv4MapLock = new LockObject("IPv4MapLock");

        internal static IPAddress LastIPv4(string hname)
        {
            IPAddress theIPAddr;
            using (new LockAndElevate(IPv4MapLock))
            {
                if (IPv4Map.ContainsKey(hname))
                    return IPv4Map[hname];
                theIPAddr = _LastIPv4(hname);
                if (theIPAddr != null)
                    IPv4Map.Add(hname, theIPAddr);
            }
            return theIPAddr;
        }

        internal static IPAddress _LastIPv4(string hname)
        {
            IPAddress ip = null;
            IPAddress[] list;
            if (IPAddress.TryParse(hname, out ip))
                return ip;
            try
            {
#if __MonoCS__
                Thread.Sleep(Isis.my_pid%1000 * 3);
#endif
                list = Dns.GetHostAddresses(hname);
            }
            catch (SocketException) { Isis.WriteLine("Warning: IP Address or hostname <" + hname + "> is unknown"); return new IPAddress(0); }
            IPAddress rval = null;
            if (IPAddress.TryParse(hname, out rval))
                return rval;
            foreach (IPAddress a in list)
            {
                if (!String.IsNullOrEmpty(hname) && !(IPAddress.TryParse(hname, out ip)) && !String.IsNullOrEmpty(ISIS_SUBNET) && !String.IsNullOrEmpty(ISIS_NETMASK))
                {
                    if (IPv4InNetwork(a, IPAddress.Parse(ISIS_NETMASK), IPAddress.Parse(ISIS_SUBNET)))
                        rval = a;
                }
                else
                {
                    if (a.AddressFamily.Equals(AddressFamily.InterNetwork))
                        rval = a;  // Picks last in the list
                }
            }
            if (rval != null)
                return rval;
            throw new IsisException("IPv6 support: not yet implemented");
        }

        private static void tryToJoin()
        {
            if ((IsisSystem.Debug & IsisSystem.STARTSEQ) != 0)
                Isis.WriteLine("Sending the Oracle a JOIN request -- myAddress " + Isis.my_address + (Isis.foundOracle == null ? "" : ", Oracle is " + Isis.foundOracle));
            int mode = Group.CREATE | Group.JOIN;
            if (ISIS_CANJOINORACLE)
                mode |= Group.CANBEORACLE;
            if (Isis.foundOracle == null || Isis.ISIS_TCP_ONLY == false)
                ORACLE.doSend(false, false, Isis.JOIN, Isis.my_address, mode, new String[] { "ORACLE" }, new Address[] { ORACLE.gaddr }, 0L, new long[] { 0 }, new int[] { 0 }, ++IsisSystem.IsisJoinCounter);
            else
                ORACLE.doP2PSend(Isis.foundOracle, true, Isis.JOIN, Isis.my_address, mode, new String[] { "ORACLE" }, new Address[] { ORACLE.gaddr }, 0L, new long[] { 0 }, new int[] { 0 }, ++IsisSystem.IsisJoinCounter);
        }

        internal static int pingCntr;
        internal static bool IWasLeader = false;

        internal static void SetupIM()
        {
            Isis.ISISMEMBERS = new Group("ISISMEMBERS");
            if (Isis.ISIS_LARGE)
                Isis.ISISMEMBERS.SetLarge();
            MCMDSocket.Setup(Isis.ISISMEMBERS);
            ISISMEMBERS.ViewHandlers *= (ViewHandler)delegate(View v)
            {
                if ((IsisSystem.Debug & (IsisSystem.TCPOVERLAY | IsisSystem.TUNNELING)) != 0)
                    Isis.WriteLine(v.ToString());
                Isis.ISIS_ASYNCMTOTALLIMIT = Math.Min(Isis.ISIS_MINASYNCMTOTAL, Math.Max(2, 1250 / v.members.Length));
                List<Group> gc = Group.IsisGroupsClone();
                foreach (Address who in v.leavers)
                {
                    ReliableSender.NodeHasFailed(who);
                    foreach (Group g in gc)
                        Group.GroupNoteFailure(g, who);
                }
                ReliableSender.CleanTCP();
                if (v.IAmLeader())
                    if (v.viewid == 0)
                        IWasLeader = true;
                    else if (!IWasLeader)
                    {
                        IWasLeader = true;
                        AnnounceMCMDMapping();
                    }
                ISISMEMBERS.OOBNewView(v);
            };
            ISISMEMBERS.doRegister(Isis.FDETECTION, (Action<Address>)delegate(Address who)
            {
                NodeHasFailed(who, "Informed by some remote caller", true);
            });
            ISISMEMBERS.doRegister(Isis.INQUIRE, (Action)delegate()
            {
                while (!IsisSystem.IsisActive)
                    Thread.Sleep(250);
                while (IsisSystem.IsisActive)
                    if (IsisSystem.waitForWorkerSetup.WaitOne(120000))
                    {
                        IsisSystem.RTS.ThreadCntrs[3]++;
                        ISISMEMBERS.doNullReply();
                        return;
                    }
            });
            ISISMEMBERS.doRegister(Isis.BECLIENT, (Action<string>)delegate(string gname)
            {
                ISISMEMBERS.doReply(Client.GetTSigs(gname));
            });
            ISISMEMBERS.doRegister(Isis.BECLIENT, (Action<string, Address>)delegate(string gname, Address newRep)
            {
                Client.ResetRep(gname, newRep);
            });
            ISISMEMBERS.doRegister(Isis.PING, (Action<string>)delegate(string message)
            {
                Isis.WriteLine("PING: [" + Isis.TimeToString(Isis.NOW()) + "]: Received <" + message + "> in view " + Isis.ISISMEMBERS.theView.viewid);
                pingCntr++;
            });
            ISISMEMBERS.SetupIMTunnels();
            if (Isis.WORKER_MODE)
                return;
            if (Isis.IAmOracle && ORACLE.theView.IAmLeader())
                ISISMEMBERS.Create();
            else
                ISISMEMBERS.Join();
        }

        internal static void AnnounceMCMDMapping()
        {
            if (Isis.ISISMEMBERS == null || !Isis.ISISMEMBERS.HasFirstView || !Isis.ISISMEMBERS.theView.IAmLeader())
                return;
            if ((IsisSystem.Debug & IsisSystem.MCMDMAP) != 0)
                Isis.WriteLine("Broadcasting new MCMD Mapping!");
            Isis.ISISMEMBERS.doSend(false, false, Isis.REMAP, Isis.MapperEpochId, MCMDSocket.nextPhysIPAddr, MCMDSocket.MCMDvirtual.GetMapAll());
        }

        internal class PendingLeaderOps
        {
            // Note that depending on which constructor is used, the DAL logic can handle reliable message forwarding OR at-most-once execution by a leader
            // Important to keep track of which case we're dealing with in the DAL code or various havoc can ensue....
            internal Group group;
            internal Address Sender;
            internal Msg reqMsg;
            internal Msg replyMsg;
            internal int uid;
            internal ThreadStart doTheAction;

            internal PendingLeaderOps(Group g, Address s, int u, ThreadStart d)
            {
                group = g;
                Sender = s;
                uid = u;
                doTheAction = d;
                reqMsg = g.getReplyToAndClear();
            }

            internal PendingLeaderOps(Group g, Address s, int u, Msg rqmsg, Msg rep)
            {
                group = g;
                Sender = s;
                uid = u;
                reqMsg = rqmsg;
                replyMsg = rep;
            }
        }

        internal static LockObject PendingLeaderOpsLock = new LockObject("PendingLeanderOpsLock");
        internal static List<PendingLeaderOps> PendingLeaderOpsList = new List<PendingLeaderOps>();

        internal static string GetPLLState()
        {
            using (new LockAndElevate(PendingLeaderOpsLock))
            {
                if (PendingLeaderOpsList.Count() == 0)
                    return "";
                string s = "List of callbacks for DoAsLeader requests:\r\n";
                foreach (PendingLeaderOps plo in PendingLeaderOpsList)
                    s += "    Group <" + plo.group.gname + ">, Sender " + plo.Sender + ", uid=" + plo.uid + "\r\n";
                return s;
            }
        }

        internal static void PendingLeaderViewChange(View v)
        {
            // If someone gets added to the ORACLE, make sure to delete them from the CanBeOracleList
            // Obviously the leader won't find them on the list, but the other group members will
            // This way when we look for a candidate to replace a departing ORACLE member, we won't get
            // confused (in the VUProtocol) by entries that aren't actually valid candidates
            if (v.joiners.Length > 0)
                using (new LockAndElevate(CanBeOracleListLock))
                    foreach (Address who in v.joiners)
                        CanBeOracleList.Remove(who);

            if (v.IAmLeader() == false)
                return;

            List<PendingLeaderOps> callbackList = new List<PendingLeaderOps>();
            using (new LockAndElevate(PendingLeaderOpsLock))
            {
                List<PendingLeaderOps> newPendingLeaderOpsList = new List<PendingLeaderOps>();
                foreach (PendingLeaderOps outerPlo in PendingLeaderOpsList)
                {
                    if (!outerPlo.group.gaddr.Equals(v.gaddr))
                        newPendingLeaderOpsList.Add(outerPlo);
                    else if (outerPlo.replyMsg != null)
                        callbackList.Add(outerPlo);
                    else if (outerPlo.replyMsg != null)
                    {
                        PendingLeaderOps plo = outerPlo;
                        Thread t = new Thread(delegate()
                        {
                            try
                            {
                                Msg.InvokeFromBArray(plo.replyMsg.payload, (Action<Address, byte, Address, byte[], int, int>)delegate(Address gaddr, byte type, Address destProc, byte[] buffer, int vid, int MsgID)
                                {
                                    Group g = Group.doLookup(gaddr);
                                    if (g != null)
                                        ReliableSender.doSend(false, ReliableSender.my_p2psocket, g, type, destProc, buffer, vid, MsgID, true, null);
                                    plo.group.doSend(false, false, DALDONE, plo.group.gaddr, plo.Sender, plo.uid);
                                });
                            }
                            catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                            IsisSystem.ThreadTerminationMagic();
                        });
                        t.Name = "DAL Worker thread spawned in PendingLeaderViewChange";
                        t.Start();
                    }
                }
                PendingLeaderOpsList = newPendingLeaderOpsList;
            }
            foreach (PendingLeaderOps outerPlo in callbackList)
            {
                PendingLeaderOps plo = outerPlo;
                Thread t = new Thread(delegate()
                {
                    try
                    {
                        Group g = Group.doLookup(plo.group.gaddr);
                        if (g == null)
                            throw new IsisException("DAL callback: group not found");
                        Msg replyTo = plo.reqMsg;
                        g.setReplyTo(plo.reqMsg);
                        if ((IsisSystem.Debug & IsisSystem.DALLOGIC) != 0)
                            Isis.WriteLine("DAL callback: In group <" + g.gname + ">, Thread DoAsLeaderCallback, setting theMsg=" + replyTo.vid + ":" + replyTo.msgid + ", needsreply=" + ((replyTo.flags & Msg.NEEDSREPLY) != 0));
                        if (plo.doTheAction != null)
                            plo.doTheAction();
                        g.clearReplyTo();
                        g.doUnorderedSend(DALDONE, g.gaddr, plo.Sender, plo.uid);
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    IsisSystem.ThreadTerminationMagic();
                });
                t.Name = "DoAsLeader Callback";
                t.Start();
            }
        }

        internal static void DALReplyNotify(Group g, Msg rmsg, PendingLeaderOps plos, Msg replyTo)
        {
            using (new LockAndElevate(g.groupLock))
                g.NotifyDALOnReply = null;
            byte f = replyTo.flags;
            replyTo.flags |= Msg.NEEDSREPLY;
            if ((IsisSystem.Debug & IsisSystem.DALLOGIC) != 0)
                Isis.WriteLine("DALReplyNotify: Doing an unordered multicast to DALdone for " + plos.group.gname + ", rqsender=" + plos.Sender +
                    ", uid=" + plos.uid + ", rmsg=" + rmsg.sender + "::" + rmsg.vid + ":" + rmsg.msgid);
            if (g != ORACLE)
                ORACLE.doUnorderedQueryToBA(Group.ALL, new Timeout(Isis.ISIS_DEFAULTTIMEOUT, Timeout.TO_FAILURE, "DALDONE"), DALDONE, Isis.my_address, plos.group.gaddr, plos.Sender, plos.uid, replyTo, rmsg, true);
            else
                ORACLE.doSend(false, false, DALDONE, Isis.my_address, plos.group.gaddr, plos.Sender, plos.uid, replyTo, rmsg, false);
            replyTo.flags = f;
        }

        // Should be called by all members of a group upon receipt of a virtually synchronous multicast containing somne request
        private static void DoAsLeader(Group g, Address Sender, int uid, ThreadStart theAction)
        {
            if ((IsisSystem.Debug & IsisSystem.DALLOGIC) != 0)
                Isis.WriteLine("DoAsLeader(<" + g.gname + ">, IAmLeader=" + g.theView.IAmLeader() + ", sender=" + Sender + ", uid=" + uid + ", action=" + theAction.Method + ")");
            PendingLeaderOps plos = new PendingLeaderOps(g, Sender, uid, theAction);
            if (!g.theView.IAmLeader())
                using (new LockAndElevate(PendingLeaderOpsLock))
                {
                    foreach (PendingLeaderOps plo in PendingLeaderOpsList)
                        if (plo.group.gaddr.Equals(g.gaddr) && plo.Sender.Equals(Sender) && plo.uid == uid)
                        {
                            if (plo.replyMsg != null)
                                return;
                            else
                                Isis.WriteLine("WARNING: DoAsLeader was called twice for the identical request!  Gaddr " + g.gaddr + ", Sender " + Sender + ", UID " + uid);
                        }
                    PendingLeaderOpsList.Add(plos);
                    return;
                }
            using (new LockAndElevate(ORACLE.groupLock))
                ORACLE.NotifyDALOnReply = plos;
            if (plos.reqMsg != null)
            {
                g.setReplyTo(plos.reqMsg);
                if ((IsisSystem.Debug & IsisSystem.DALLOGIC) != 0)
                    Isis.WriteLine("DAL: In group <" + g.gname + ">, setting theMsg=" + plos.reqMsg.vid + ":" + plos.reqMsg.msgid + ", needsreply=" + ((plos.reqMsg.flags & Msg.NEEDSREPLY) != 0));
            }
            plos.doTheAction();
            if (plos.reqMsg != null && (plos.reqMsg.flags & Msg.NEEDSREPLY) != 0 && g.getReplyTo() != null)
            {
                if ((IsisSystem.Debug & IsisSystem.DALLOGIC) != 0)
                    Isis.WriteLine("DoAsLeader: Sending NullReply() because the dal action routine failed to send a reply");
                g.NullReply();
            }
            g.clearReplyTo();
            // Notice the (legitimate) race condition here.  If the DoAsLeader logic doesn't send a reply, and JOIN doesn't do so,
            // then if the leader finishes the operation but then fails before this next Send occurs, the request will be repeated
            // by the new leader.  JOIN happens to be idempotent, which solves this particular problem.  If a request isn't
            // idempotent it really MUST have a requested reply, even if the reply is just "OK" and will be ignored
            g.doUnorderedSend(DALDONE, g.gaddr, Sender, uid);
        }

        internal static int OracleJoinsUnderway;
        internal static LockObject CanBeOracleListLock = new LockObject("CanBeOracleListLock");
        internal static List<Address> CanBeOracleList = new List<Address>();
        internal static List<Address> FDRunning = new List<Address>();

        private static void SetupORACLE()
        {
            ORACLE.RegisterViewHandler((ViewHandler)PendingLeaderViewChange);
            ORACLE.doRegister(DALDONE, (Action<Address, Address, Address, int, Msg, Msg, bool>)delegate(Address sender, Address gaddr, Address rqInitiator, int uid, Msg rqmsg, Msg rmsg, bool needsReply)
            {
                if (sender.isMyAddress())
                    return;
                if ((IsisSystem.Debug & IsisSystem.DALLOGIC) != 0)
                    Isis.WriteLine("DALdone: Received a DAL done(1) event from " + sender + " for " + gaddr + ", sender=" + rqInitiator + ", uid=" + uid);
                using (new LockAndElevate(PendingLeaderOpsLock))
                {
                    bool fnd = false;
                    foreach (PendingLeaderOps plo in PendingLeaderOpsList)
                        if (plo.group.gaddr.Equals(gaddr) && plo.Sender.Equals(rqInitiator) && plo.uid == uid)
                        {
                            plo.replyMsg = rmsg;
                            fnd = true;
                            break;
                        }
                    if (!fnd)
                    {
                        // Occurs if there is a race between the original request to the ORACLE and the Reply by the initial leader
                        if ((IsisSystem.Debug & IsisSystem.DALLOGIC) != 0)
                            Isis.WriteLine("DoAsLeader(<ORACLE>, sender=" + sender + ", uid=" + uid + "), got a DONE message but didn't find a PLOS record, create one");
                        PendingLeaderOpsList.Add(new PendingLeaderOps(ORACLE, rqInitiator, uid, rqmsg, rmsg));
                    }
                }
                if (needsReply)
                    ORACLE.doReply("OK");
            });
            ORACLE.doRegister(DALDONE, (Action<Address, Address, int>)delegate(Address gaddr, Address sender, int uid)
            {
                if ((IsisSystem.Debug & IsisSystem.DALLOGIC) != 0)
                    Isis.WriteLine("DALdone(2): sender " + sender + " gaddr " + gaddr + ", uid=" + uid);
                using (new LockAndElevate(PendingLeaderOpsLock))
                {
                    List<PendingLeaderOps> newPendingLeaderOpsList = new List<PendingLeaderOps>();
                    foreach (PendingLeaderOps plo in PendingLeaderOpsList)
                        if (!plo.group.gaddr.Equals(gaddr) || !plo.Sender.Equals(sender) || plo.uid != uid)
                            newPendingLeaderOpsList.Add(plo);
                        else if ((IsisSystem.Debug & IsisSystem.DALLOGIC) != 0)
                            Isis.WriteLine("DALdone(2): Remove DAL record for " + sender + " for " + gaddr + ", uid=" + uid);
                    PendingLeaderOpsList = newPendingLeaderOpsList;
                }
            });

            ORACLE.doRegister(JOIN, (Action<Address>)delegate(Address who)
            {
                if ((IsisSystem.Debug & IsisSystem.STARTSEQ) != 0)
                    Isis.WriteLine("Heard from the ORACLE, his address is " + who);
                Isis.foundOracle = who;
                // This actually prevents ME from dinging HIM as faulty
                ReliableSender.nodeInStartup(who);
            });

            ORACLE.doRegister(ORACLERUNNING, (Action<Address>)delegate(Address who)
            {
                if (!IsisSystem.IsisActive || !ORACLE.IAmLeader() || my_address.CompareTo(who) <= 0)
                    return;
                if (ISISMEMBERS.HasFirstView)
                {
                    ISISMEMBERS.doSend(false, false, PARTITIONED, who);
                    Thread.Sleep(1000);
                    throw new IsisException("ORACLE: Discovered I am in a minority partition" + IsisSystem.GetState());
                }
            });

            ORACLE.doRegister(JOIN, (Action<Address, int, String[], Address[], long, long[], int[], int>)delegate(Address who, int mode, String[] gnames, Address[] gaddrs, long offset, long[] tsigs, int[] flags, int uid)
            {
                // JOIN actually uses a different scheme to ensure fault-tolerance, implemented by the VUProtocol
                // In future work I really should merge these into the DoAsLeader pattern but there are some subtle
                // issues because of the way that group view events in the ORACLE itself need to be handled
                if (gaddrs.Length == 1 && gaddrs[0].Equals(ORACLE.gaddr) && ORACLE.theView != null)
                {
                    if (ORACLE.theView.GetRawRankOf(who) != -1)
                        // Ignore artifacts of the start sequence, which can involve asking to join multiple times
                        return;

                    if (ORACLE.theView.GetMyRank() != -1)
                    {
                        // Inhibit creation of a second oracle, in case leader is slow to respond
                        if (!who.isMyAddress())
                        {
                            if ((IsisSystem.Debug & IsisSystem.STARTSEQ) != 0)
                                Isis.WriteLine("Received a JOIN inquiry in ORACLE, sending a message to inhibit creation of new ORACLEs to " + who);
                            Isis.ORACLE.doPureP2PSend(who, true, Isis.JOIN, Isis.my_address);
                            // Deals with sluggish C# class loader, which can lock out I/O for many seconds at a time during startup
                            ReliableSender.nodeInStartup(who);
                        }
                    }
                }

                if ((IsisSystem.Debug & (IsisSystem.STARTSEQ | IsisSystem.GROUPEVENTS | IsisSystem.VIEWCHANGE)) != 0)
                {
                    string gns = "", isls = " ";
                    foreach (string s in gnames)
                        gns += " " + s;
                    foreach (int f in flags)
                        isls += f + " ";
                    Isis.WriteLine("Oracle received a JOIN <" + gns + " >, gaddrs=" + Address.VectorToString(gaddrs) + ", flags={" + isls + "}, request... my_address " + Isis.my_address + ", joiner address " + who);
                }
                if (gnames.Length == 1 && gnames[0].Equals("ORACLE"))
                {
                    if (who.isMyAddress())
                        return;
                    using (new LockAndElevate(recent_inquiries_lock))
                    {
                        foreach (Address a in recent_inquiries)
                            if (a.Equals(who))
                                return;
                        recent_inquiries.Add(who);
                        // Using...
                        {
                            Address ri = who;
                            Isis.OnTimer(5000, (TimerCallback)delegate() { using (new LockAndElevate(recent_inquiries_lock)) recent_inquiries.Remove(ri); });
                        }
                    }
                    bool coreOracleJoiner = false, amACandidate = false;
                    using (new LockAndElevate(Isis.ORACLE.groupLock))
                    {
                        if (ORACLE.HasFirstView && (mode & Group.CANBEORACLE) != 0)
                        {
                            if (ORACLE.theView.nLive() < ISIS_ORACLESIZE - OracleJoinsUnderway)
                            {
                                // If we get here, we're currently short on ORACLE members and had better do something about it
                                if (ISIS_HOSTS_IPADDRS == null)
                                {
                                    ++OracleJoinsUnderway;
                                    coreOracleJoiner = true;
                                }
                                else foreach (IPAddress ipa in ISIS_HOSTS_IPADDRS)
                                        if (ipa.Equals(who.home) == true)
                                        {
                                            ++OracleJoinsUnderway;
                                            coreOracleJoiner = true;
                                            break;
                                        }
                            }
                            else if (ORACLE.IAmLeader())
                                amACandidate = true;
                        }
                    }
                    if (amACandidate)
                        // Pass the word: this is a candidate to join the oracle if the need ever arises
                        ORACLE.Send(Isis.CANBEORACLE, who);

                    if (coreOracleJoiner == false)
                    {
                        if (!ORACLE.IAmLeader())
                            return;
                        SendInitialOracleLeaderInfo(who, Isis.my_address);
                        return;
                    }
                }
                if ((IsisSystem.Debug & (IsisSystem.STARTSEQ | IsisSystem.GROUPEVENTS)) != 0)
                    Isis.WriteLine("Initiating VUProtocol in ORACLE.Join");
                bool fnd = false;
                foreach (string gn in gnames)
                    if (Group.TrackingProxyLookup(gn) != null)
                    {
                        fnd = true;
                        break;
                    }
                if (!fnd)
                    foreach (Address ga in gaddrs)
                        if (Group.TrackingProxyLookup(ga) != null)
                        {
                            fnd = true;
                            break;
                        }
                if (fnd) new Thread(delegate()
                {
                    Thread.CurrentThread.Name = "MCMDSocket.InitializeMap";
                    try
                    {
                        MCMDSocket.InitializeMap(who, gnames, gaddrs);
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    IsisSystem.ThreadTerminationMagic();
                }).Start();
                VUProtocol(JOIN, who, mode, gnames, gaddrs, offset, tsigs, flags, uid);
            });

            ORACLE.RegisterHandler(Isis.CANBEORACLE, (Action<Address>)delegate(Address who)
            {
                using (new LockAndElevate(CanBeOracleListLock))
                    CanBeOracleList.Add(who);
            });

            ORACLE.doRegister(LEAVE, (Action<Address, int, String[], Address[], int[], int>)delegate(Address who, int mode, String[] gnames, Address[] gaddrs, int[] flags, int uid)
            {
                if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                    Isis.WriteLine("Initiating VUProtocol in ORACLE.Leave");
                VUProtocol(LEAVE, who, mode, gnames, gaddrs, 0L, null, flags, uid);
            });

            ORACLE.doRegister(TERMINATE, (Action<Address, int, Address[]>)delegate(Address who, int uid, Address[] gaddrs)
            {
                List<Group> glist = new List<Group>();
                List<Group> lglist = new List<Group>();
                foreach (Address a in gaddrs)
                {
                    Group g = Group.TrackingProxyLookup(a);
                    if (g == null)
                        continue;
                    if ((g.flags & Group.G_ISLARGE) == 0)
                        glist.Add(g);
                    else
                        lglist.Add(g);
                }
                if (glist.Count() != 0)
                    Group.doMultiSend(glist, true, TERMINATE);
                if (lglist.Count() != 0)
                    foreach (Group g in lglist)
                        g.P2PSend(g.theView.members[0], Isis.RELAYSEND, new Msg(TERMINATE));
                foreach (Address a in gaddrs)
                {
                    Group g = Group.TrackingProxyLookup(a);
                    if (g == null)
                        continue;
                    using (new LockAndElevate(Group.TPGroupsLock))
                        Group.TPGroups.Remove(g.gaddr);
                }
            });

            ORACLE.doRegister(Isis.RELAYJOIN, (Action<Address, int, int, string[], Address[], long, long[], int[]>)delegate(Address joiner, int uid, int mode, string[] gnames, Address[] gaddrs, long offset, long[] tsigs, int[] flags)
            {
                Isis.DoAsLeader(ORACLE, joiner, uid, (ThreadStart)delegate()
                {
                    if ((IsisSystem.Debug & IsisSystem.RELAYLOGIC) != 0)
                    {
                        Isis.WriteLine("In DAL for Isis.RELAYJOIN: sender " + joiner + " vectors of length " + gnames.Length);
                        for (int n = 0; n < gnames.Length; n++)
                            Isis.WriteLine("   gname " + gnames[n] + ", gaddrs " + gaddrs[n] + ", gsigs " + tsigs[n]);
                    }
                    ORACLE.doSend(false, false, Isis.JOIN, joiner, mode, gnames, gaddrs, offset, tsigs, flags, ++IsisSystem.IsisJoinCounter);
                    ORACLE.doReply("OK");
                });
            });

            ORACLE.doRegister(Isis.RELAYLEAVE, (Action<Address, int, string[], Address[], int[]>)delegate(Address sender, int uid, string[] gnames, Address[] gaddrs, int[] flags)
            {
                Isis.DoAsLeader(ORACLE, sender, uid, (ThreadStart)delegate()
                {
                    if ((IsisSystem.Debug & IsisSystem.RELAYLOGIC) != 0)
                    {
                        Isis.WriteLine("In Isis.RELAYLEAVE: sender " + sender + " vectors of length " + gnames.Length);
                        for (int n = 0; n < gnames.Length; n++)
                            Isis.WriteLine("   gname " + gnames[n] + ", gaddrs " + gaddrs[n]);
                    }
                    ORACLE.doSend(false, false, Isis.LEAVE, sender, 0, gnames, gaddrs, flags, ++IsisSystem.IsisJoinCounter);
                    ORACLE.doReply("OK");
                });
            });


            ORACLE.doRegister(Isis.RELAYTERM, (Action<Address, int, Address[]>)delegate(Address who, int uid, Address[] gaddrs)
            {
                Isis.DoAsLeader(ORACLE, who, uid, (ThreadStart)delegate()
                {
                    ORACLE.doSend(false, false, Isis.TERMINATE, Isis.my_address, ORACLE.uids++, gaddrs);
                    ORACLE.doReply("OK");
                });
            });

            ORACLE.doRegister(Isis.ISSTABLE, (Action<Address, int>)delegate(Address gaddr, int upTo)
            {
                Isis.clearOldVDS(gaddr, upTo);
            });

            ORACLE.doRegister(FDETECTION, (Action<Address>)delegate(Address who)
            {
                // Another special situation since failure may have caused the prior leader to crash
                // Todo: Look for a way to fold this into the DoAsLeader pattern used above
                // For now the needed pattern is implemented separately in the VUProtocol logic
                List<Group> glist = new List<Group>(), theClone = Group.IsisAllGroupsClone(false);
                bool IwasOldLeader = ORACLE.LeaderMode, IamNewLeader = false;
                if ((IsisSystem.Debug & (IsisSystem.GROUPEVENTS | IsisSystem.FAILURES)) != 0)
                    Isis.WriteLine("FDETECTION message received for " + who + "(I was old leader: " + IwasOldLeader + ")");
                if (who.isMyAddress())
                    IsisSystem.GotPoison("Failure detection broadcast reported my demise");
                if (!IsisSystem.IsisActive || IsisSystem.IsisRestarting || ISISMEMBERS == null || !ISISMEMBERS.HasFirstView)
                    return;
                ReliableSender.NodeHasFailed(who);
                foreach (Group g in theClone)
                {
                    using (new ILock(ILock.LLBRIEF, g.gaddr))
                    {
                        if (g.theView == null || g.theView.GetRawRankOf(who) == -1)
                            continue;
                        if ((IsisSystem.Debug & (IsisSystem.GROUPEVENTS | IsisSystem.FAILURES)) != 0)
                            Isis.WriteLine("FDETECTION(Isis_Groups) calling View.NoteFailed in <" + g.gname + "> for " + who);
                        View.noteFailed(g, who);
                        glist.Add(g);
                        if (g == ORACLE)
                            if (IwasOldLeader == false && g.theView.IAmLeader() == true)
                            {
                                if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                                    Isis.WriteLine("I am new leader!");
                                IamNewLeader = true;
                            }
                            else if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                                Isis.WriteLine("In scan of g.ORACLE I am NOT the new leader because IwasOldLeader=" + IwasOldLeader + " and g.theView.IAmLeader=" + g.theView.IAmLeader() + " in view " + g.theView);
                    }
                }
                if (Isis.ClientOf == null && ORACLE.IAmLeader())
                {
                    View v;
                    using (new LockAndElevate(ORACLE.ViewLock))
                        v = ORACLE.theView;
                    // Break any wait states within the ORACLE itself.  This avoids deadlock
                    foreach (Address a in v.members)
                        ORACLE.doP2PSend(a, true, FANNOUNCE, who);
                }
                // Now send a virtually synchronous announcement but do it in a separate thread
                new Thread(delegate()
                {
                    Thread.CurrentThread.Name = "Send FANNOUNCE " + who;
                    try { ISISMEMBERS.Send(FANNOUNCE, who); }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    IsisSystem.ThreadTerminationMagic();
                }).Start();
                if (Isis.ClientOf != null)
                    return;
                // Code run only by members of the ORACLE group
                if (glist.Count() > 0)
                    VUProtocol(LEAVE, who, 0, ToDeDuppedArray(glist.Select(g => g.gname).ToList<string>()), ToDeDuppedArray(glist.Select(g => g.gaddr).ToList<Address>()), 0L, null, null, -1);
                if (!IamNewLeader || OracleViewTaskRunning || Isis.RunningLeaderLogic)
                    return;
                Isis.RunningLeaderLogic = true;
                Isis.LeaderId += ORACLE.theView.GetMyRank();
                ReliableSender.SendPoison(who, Isis.my_address + " believes that you have failed");
                new Thread(delegate()
                {
                    try
                    {
                        Dictionary<Address, bool> OnceAndFutureOracleMembers = new Dictionary<Address, bool>();
                        foreach (Address a in ORACLE.theView.members)
                            OnceAndFutureOracleMembers.Add(a, false);
                        int scannedTo = -1;
                        if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                            Isis.WriteLine("Sending INQUIRE message...");
                        ORACLE.theView.isFinal = false;
                        List<Address> mustContact = new List<Address>();
                        ORACLE.doQueryInvoke(Group.ALL, new Timeout(Isis.ISIS_DEFAULTTIMEOUT * 20, Timeout.TO_FAILURE, "INQUIRE"), INQUIRE, my_address, (MergeProposals)delegate(Address[] from, Address[] hisLeader, ViewDelta[][] vds)
                        {
                            doMergeProposals(from, hisLeader, vds, OnceAndFutureOracleMembers, mustContact, ref scannedTo);
                        });
                        while (mustContact.Count() > 0)
                        {
                            List<Address> contacting = mustContact;
                            mustContact = new List<Address>();
                            foreach (Address a in contacting)
                            {
                                List<Address> whoReplied = new List<Address>();
                                List<Address> hisLeader = new List<Address>();
                                List<ViewDelta[]> vds = new List<ViewDelta[]>();
                                ISISMEMBERS.doP2PQuery(a, new Timeout(Isis.ISIS_DEFAULTTIMEOUT * 20, Timeout.TO_FAILURE, "INQUIRE"), INQUIRE, my_address, whoReplied, hisLeader, vds);
                                if (whoReplied.Count() > 0)
                                    doMergeProposals(whoReplied.ToArray(), hisLeader.ToArray(), vds.ToArray(), OnceAndFutureOracleMembers, mustContact, ref scannedTo);
                            }
                        }
                        int rCnt = 0;
                        foreach (KeyValuePair<Address, bool> kvp in OnceAndFutureOracleMembers)
                            if (kvp.Value)
                                ++rCnt;
                        if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                        {
                            string resps = " ";
                            foreach (KeyValuePair<Address, bool> kvp in OnceAndFutureOracleMembers)
                                resps += "<" + kvp.Key + "::" + kvp.Value + "> ";
                            Isis.WriteLine("After INQUIRE: Contacted " + resps + " total of " + rCnt + " replies, needed " + (OnceAndFutureOracleMembers.Count() + 1) / 2);
                            Isis.WriteLine("After INQUIRE proposed contains...");
                            if (Isis.Proposed == null)
                                Isis.WriteLine(" ... Isis.proposed is null");
                            else
                                for (int p = 0; p < Isis.Proposed.Length; p++)
                                    Isis.WriteLine("PROPOSAL[" + p + "]=" + Isis.Proposed[p]);
                            Isis.WriteLine("Starting the OracleViewTask thread");
                        }
                        if (rCnt < (OnceAndFutureOracleMembers.Count() + 1) / 2)
                            throw new IsisException("New ORACLE leader was unable to contact a quorum of once and future ORACLE members");
                        // Runs only in the current leader... and now, that's me!
                        ORACLE.LeaderMode = true;
                        ORACLE.TakingOver = true;
                        Isis.OracleViewThread = new Thread(Isis.OracleViewTask);
                        Isis.IAmOracle = true;
                        Isis.OracleViewThread.Name = "Isis <ORACLE> View Task";
                        Isis.OracleViewThread.Start();
                        MCMDSocket.RunMappingTask();
                        ISISMEMBERS.Send(BECLIENT, Isis.my_address);
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    IsisSystem.ThreadTerminationMagic();
                }).Start();
            });
            ORACLE.doRegister(BECLIENT, (Action<string>)delegate(string gname)
            {
                ORACLE.doReply(Client.SelectHisRep(gname));
            });
            ORACLE.doRegister(RELAYREGISTERVG, (Action<Address, Address, Address[]>)delegate(Address sender, Address nVGA, Address[] members)
            {
                ORACLE.doSend(false, false, REGISTERVG, sender, nVGA, members);
            });
            ORACLE.doRegister(REGISTERVG, (Action<Address, Address, Address[]>)delegate(Address sender, Address nVGA, Address[] members)
            {
                Group.noteVGMap(sender, nVGA, members);
            });
            ORACLE.GroupOpen = ORACLE.WasOpen = true;
            ReliableSender.StartGroupReader(ORACLE);
        }

        // O(n^2) but the list should be very short...
        private static string[] ToDeDuppedArray(List<string> slist)
        {
            List<string> deDupped = new List<string>();
            foreach (string s in slist)
            {
                bool fnd = false;
                foreach (string ds in deDupped)
                {
                    if (ds.Equals(s))
                        fnd = true;
                    break;
                }
                if (!fnd)
                    deDupped.Add(s);
            }
            return deDupped.ToArray();
        }

        private static Address[] ToDeDuppedArray(List<Address> alist)
        {
            List<Address> deDupped = new List<Address>();
            foreach (Address a in alist)
            {
                bool fnd = false;
                foreach (Address da in deDupped)
                {
                    if (da.Equals(a))
                        fnd = true;
                    break;
                }
                if (!fnd)
                    deDupped.Add(a);
            }
            return deDupped.ToArray();
        }

        // As we INQUIRE we learn about new proposals, which we merge into the existing list.  We may also encounter added group members
        // Handles the so-called "dueling leaders" situation, in which there are two proposals for the same slot
        internal static void doMergeProposals(Address[] from, Address[] hisLeader, ViewDelta[][] vds, Dictionary<Address, bool> OnceAndFutureOracleMembers, List<Address> mustContact, ref int scannedTo)
        {
            Isis.ViewDelta[] proposed = Isis.Proposed;
            if (proposed == null)
                proposed = new Isis.ViewDelta[0];
            for (int v = 0; v < from.Length; v++)
            {
                if (OnceAndFutureOracleMembers.ContainsKey(from[v]))
                    OnceAndFutureOracleMembers[from[v]] = true;
                else
                    throw new IsisException("Unexpected ORACLE INQUIRY response: from " + from[v] + " but I didn't think I had contacted him!");
                ViewDelta[] vd = vds[v];
                if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                {
                    Isis.WriteLine("Recieved INQUIRE replies...");
                    for (int vv = 0; vv < vd.Length; vv++)
                        Isis.WriteLine("REPLY[" + v + ":" + vv + "]=" + vd[vv]);
                }
                bool useProposal = false, ignoreIt = false;
                int minLen = Math.Min(vd.Length, proposed.Length);
                for (int i = 0; i < minLen; i++)
                {
                    if (vd[i].leaderId > Isis.Proposed[i].leaderId)
                    {
                        // Dueling leaders (very rare): in this case we just encountered a "better" proposal
                        // We'll switch to it even if it is shorter
                        if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                            Isis.WriteLine("Dueling leaders!  Found a better proposal, switching to it");
                        useProposal = true;
                    }
                    else if (vd[i].leaderId < Isis.Proposed[i].leaderId)
                    {
                        // Dueling leaders (very rare): in this case wejust encountered an old "stale" proposal
                        // We'll switch to it even if it is shorter
                        if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                            Isis.WriteLine("Dueling leaders!  New proposal is inferior, ignoring it");
                        ignoreIt = true;
                        break;
                    }
                    if (i > scannedTo && vd[i].gaddr.Equals(ORACLE.gaddr) && vd[i].joiners.Length > 0)
                    {
                        scannedTo = i;
                        foreach (Address a in vd[i].joiners)
                            if (!OnceAndFutureOracleMembers.ContainsKey(a))
                            {
                                OnceAndFutureOracleMembers.Add(a, false);
                                mustContact.Add(a);
                            }
                    }
                }
                if (!ignoreIt && (Isis.Proposed == null || vd.Length > Isis.Proposed.Length || useProposal))
                    Isis.Proposed = vd;
            }
        }

        internal static void SendInitialOracleLeaderInfo(Address who, Address OracleLeader)
        {
            if (who.isMyAddress() || Isis.ORACLE.theView.GetRankOf(who) != -1)
                return;
            if ((IsisSystem.Debug & (IsisSystem.STARTSEQ | IsisSystem.VIEWCHANGE | IsisSystem.GROUPEVENTS)) != 0)
                Isis.WriteLine("Oracle leader sending a NULL INITIALVIEW message to a future CLIENT, my_address " + Isis.my_address + ", new client is " + who + ", ORACLE view " + ORACLE.theView);
            ORACLE.doP2PSend(who, true, INITIALVIEW, OracleLeader);
        }

        internal class GVEvent
        {
            internal int request;
            internal Address who;
            internal int mode;
            internal int uid;
            internal String[] gnames;
            internal Address[] gaddrs;
            internal long offset;
            internal long[] tsigs;
            internal int[] flags;

            internal GVEvent(int r, Address a, int m, String[] gns, Address[] gs, long off, long[] ts, int[] fl, int u)
            {
                request = r;
                who = a;
                mode = m;
                gnames = gns;
                gaddrs = gs;
                offset = off;
                tsigs = ts;
                flags = fl;
                uid = u;
            }

            public override string ToString()
            {
                string gs = " ";
                foreach (string s in gnames)
                    gs += s + " ";
                return "GVE: request=" + rToString(request) + ", who=" + who + ", mode=" + mode + ", gnames={" + gs + "}, gaddrs=" + Address.VectorToString(gaddrs) + ", flags=" + flags + ", uid=" + uid;
            }
        }

        internal static List<GVEvent> GVEList = new List<GVEvent>();
        internal static LockObject GVELock = new LockObject("GVELock");
        internal static List<GVEvent> AGVEList = new List<GVEvent>();
        internal static LockObject AGVELock = new LockObject("AGVELock");

        internal static string GetGVEState()
        {
            string s = "Group view events list:\r\n";
            using (new LockAndElevate(GVELock))
                if (GVEList.Count() > 0)
                {
                    foreach (GVEvent gve in GVEList)
                    {
                        string gns = " ";
                        if (gve.gnames != null)
                            foreach (string gs in gve.gnames)
                                gns += "<" + gs + ">";
                        s += "  Action[" + gve.uid + "]: " + rToString(gve.request) + " " + gve.who + " on groups {" + gns + "}, gaddrs {" + Address.VectorToString(gve.gaddrs) + "}\r\n";
                    }
                }
            //Omit unless debugging the GVE logic
            if ((IsisSystem.Debug & IsisSystem.GVELOGIC) != 0)
                using (new LockAndElevate(AGVELock))
                    if (AGVEList.Count() > 0)
                    {
                        foreach (GVEvent gve in AGVEList)
                        {
                            string gns = " ";
                            if (gve.gnames != null)
                                foreach (string gs in gve.gnames)
                                    gns += gs;
                            s += "  Anti-action[" + gve.uid + "]: " + gve.who + "\r\n";
                        }
                    }
            return s;
        }

        /// <exclude>
        /// <summary>
        /// Internal
        /// </summary>
        /// </exclude>
        [AutoMarshalled]
        public class ViewDelta
        {
            public string gname = "";
            public Address gaddr;
            public long tsig;
            public long leaderId;
            public int[] mcmdmap;
            public int prevVid;
            public bool isLarge;
            public int[] lastSeqns;     // i'th element gives the final incoming message count from the i'th member of the previous view
            public Address[] joiners;
            public long offset;
            public Address[] leavers;

            // Used by AutoMarshaller
            public ViewDelta()
            {
            }

            internal ViewDelta(string name, Address ga, long ts, int[] mm, int v, int nm, Address[] wantJoin, long off, Address[] wantLeave, bool lf)
            {
                leaderId = Isis.LeaderId;
                gname = name; gaddr = ga; tsig = ts; mcmdmap = mm; prevVid = v; joiners = wantJoin; offset = off; leavers = wantLeave; isLarge = lf;
                if (isLarge)
                    lastSeqns = new int[0];
                else
                    lastSeqns = new int[nm];   // These won't be final until the COMMIT event
            }

            internal ViewDelta(string name, Address g, long off, int[] mm, int v, int[] ls, bool lf)
            {
                leaderId = Isis.LeaderId;
                gaddr = g; mcmdmap = mm; prevVid = v; lastSeqns = ls; isLarge = lf; offset = off;
                joiners = leavers = new Address[0];
            }

            public bool Equals(ViewDelta vd)
            {
                return gaddr.Equals(vd.gaddr) && prevVid == vd.prevVid;
            }

            public override string ToString()
            {
                return " LeaderId=" + leaderId + ", Group <" + gname + "> " + gaddr + " (mmap " + MCMDSocket.PMCAddr(mcmdmap[0]) + ":" + MCMDSocket.PMCAddr(mcmdmap[1]) + "), isLarge=" + isLarge + ", prevVid " + prevVid + idsToVec() +
                    ", Joining: {" + Address.VectorToString(Expand(joiners)) + "}, Leaving: {" + Address.VectorToString(Expand(leavers)) + "}";
            }

            private string idsToVec()
            {
                if (lastSeqns == null || lastSeqns.Length == 0)
                    return "";
                string s = ", final msg counts: {";
                foreach (int i in lastSeqns)
                    s = s + " " + i + " ";
                return s + "}";
            }
        }

        [AutoMarshalled]
        public class UnstableList
        {
            public Address gaddr;
            public Address flusher;
            public Address sender;
            public int vid;
            public int mid_low;
            public int mid_hi;

            public UnstableList()
            {
                // Used by AutoMarshaller
            }

            internal UnstableList(Address g, Address f, Address s, int v, int ml, int mh)
            {
                gaddr = g; flusher = f; sender = s; vid = v; mid_low = ml; mid_hi = mh;
            }

            public bool Equals(UnstableList ul)
            {
                return gaddr.Equals(ul.gaddr) && sender.Equals(ul.sender) && vid == ul.vid && mid_low == ul.mid_low && mid_hi == ul.mid_hi;
            }

            public override string ToString()
            {
                return " Group " + gaddr + ", Flusher " + flusher + ": MSG[sender " + sender + ", ID " + vid + ":(" + mid_low + "-" + mid_hi + ")]";
            }
        }

        delegate void MergeProposals(Address[] who, Address[] hisLeader, ViewDelta[][] vds);

        internal static void VUProtocol(int request, Address who, int mode, String[] gnames, Address[] gaddrs, long offset, long[] tsigs, int[] flags, int uid)
        {
            if (uid != -1)
            {
                using (new LockAndElevate(AGVELock))
                    foreach (GVEvent agve in AGVEList)
                        if (agve.who.Equals(who) && agve.uid == uid)
                        {
                            if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC)) != 0)
                                Isis.WriteLine("Found an old ANTI-GVE entry... ignoring this VUProtocol request");
                            AGVEList.Remove(agve);
                            return;
                        }
            }
            if (gnames == null)
                gnames = new string[0];
            if (gaddrs == null)
                gaddrs = new Address[0];
            using (new LockAndElevate(GVELock))
            {
                bool fnd = false;
                GVEvent gve = new GVEvent(request, who, mode, gnames, gaddrs, offset, tsigs, flags, uid);
                foreach (GVEvent oldgve in GVEList)
                    if (oldgve.who.Equals(who) && oldgve.mode == mode && oldgve.gnames.Length == gnames.Length)
                    {
                        for (int i = 0; i < gnames.Length; i++)
                            if (oldgve.gnames[i].Equals(gnames[i]) && oldgve.gaddrs[i].Equals(gaddrs[i]) &&
                                (oldgve.tsigs == null || tsigs == null) ? oldgve.tsigs == tsigs : oldgve.tsigs[i] == tsigs[i])
                            {
                                fnd = true;
                                break;
                            }
                        if (fnd)
                            fnd = true;
                    }
                if (!fnd)
                {
                    if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC)) != 0)
                    {
                        string gs = " ", tss = " ", fs = " ";
                        foreach (string s in gnames)
                            gs += s + " ";
                        if (tsigs != null)
                            foreach (int ts in tsigs)
                                tss += ts + " ";
                        if (flags != null)
                            foreach (int fl in flags)
                                fs += fl + " ";
                        Isis.WriteLine("Creating a GVE entry for request " + request + ", mode " + mode + ", address " + who + ", uid " + uid +
                            ", gnames [" + gs + "], groups " + Address.VectorToString(gaddrs) + ", ts={" + tss + "}, fs={" + fs + "}");
                    }
                    GVEList.Add(gve);
                }
                else if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC)) != 0)
                    Isis.WriteLine("Found an identical GVE entry... ignoring this VUProtocol request");
            }
            if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC)) != 0)
                Isis.WriteLine("VUProtocol: BarrierRelease for LLWAIT/LGVEUPDATE");
            ILock.Barrier(ILock.LLWAIT, ILock.LGVEUPDATE).BarrierRelease(1);
        }

        internal static void PurgeGVE(Address who, int uid)
        {
            if (uid == -1 || ORACLE.theView.IAmLeader())
                return;
            using (new LockAndElevate(GVELock))
                foreach (GVEvent gve in GVEList)
                    if (who.Equals(gve.who) && uid == gve.uid)
                    {
                        GVEList.Remove(gve);
                        return;
                    }
            if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC)) != 0)
                Isis.WriteLine("... didn't find the GVE entry, creating an ANTI-GVE record");
            GVEvent agve = new GVEvent(0, who, 0, null, null, 0L, null, null, uid);
            using (new LockAndElevate(AGVELock))
                AGVEList.Add(agve);
            Isis.OnTimer(120000, (TimerCallback)delegate()
            {
                using (new LockAndElevate(AGVELock))
                {
                    foreach (GVEvent gve in AGVEList)
                        if (gve == agve)
                        {
                            if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC)) != 0)
                                Isis.WriteLine("... after 120s delay, removing an ANTI-GVE record");
                            AGVEList.Remove(agve);
                            return;
                        }
                }
            });
        }

        // Only runs in the leader
        static private bool OracleViewTaskRunning = false;
        static private Semaphore OracleBeaconTaskWait = new Semaphore(0, int.MaxValue);
        static private LockObject OracleViewTaskLock = new LockObject("OracleViewTaskLock");
        internal static void OracleViewTask()
        {
            using (new LockAndElevate(OracleViewTaskLock))
            {
                if (OracleViewTaskRunning)
                    return;
                OracleViewTaskRunning = true;
            }
            Thread OracleBeaconTask = new Thread(delegate()
            {
                while (!IsisSystem.IsisActive)
                    Thread.Sleep(250);
                try
                {
                    while (IsisSystem.IsisActive)
                    {
                        IsisSystem.RTS.ThreadCntrs[4]++;
                        // Once every 30 seconds, announce that I am an Oracle leader
                        OracleBeaconTaskWait.WaitOne(30 * 1000);
                        ORACLE.doSendRaw(Isis.ORACLERUNNING, Isis.my_address);
                    }
                }
                catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                IsisSystem.ThreadTerminationMagic();
            });
            OracleBeaconTask.Name = "OracleBeaconTask";
            OracleBeaconTask.Start();
            try
            {
                while (!IsisSystem.IsisActive || Isis.ISISMEMBERS == null)
                    Thread.Sleep(250);
                List<GVEvent> gveList = new List<GVEvent>();
                while (IsisSystem.IsisActive)
                {
                    IsisSystem.RTS.ThreadCntrs[5]++;
                    GVEvent gve;
                    if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC)) != 0)
                        Isis.WriteLine("OracleViewTask: Before BarrierWait for LLWAIT/LGVEUPDATE");
                    ILock.Barrier(ILock.LLWAIT, ILock.LGVEUPDATE).BarrierWait();
                    if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC)) != 0)
                        Isis.WriteLine("OracleViewTask: After BarrierWait for LLWAIT/LGVEUPDATE");
                    using (new LockAndElevate(GVELock))
                    {
                        if (GVEList.Count() == 0)
                            continue;
                        gve = GVEList.First();
                        GVEList.Remove(gve);
                        gveList.Add(gve);
                        bool alreadyExists = true;
                        foreach (Address gaddr in gve.gaddrs)
                        {
                            Group tpg = Group.TrackingProxyLookup(gaddr);
                            if (tpg == null || !tpg.HasFirstView)
                                alreadyExists = false;
                        }
                        if (alreadyExists)
                        {
                            foreach (GVEvent gve2 in GVEList)
                                if (Address.SameAddrs(gve.gaddrs, gve2.gaddrs) && Address.SameNames(gve.gnames, gve2.gnames))
                                {
                                    gveList.Add(gve2);
                                }
                            foreach (GVEvent gve2 in gveList)
                                if (gve2 != gve)
                                    GVEList.Remove(gve2);
                        }
                    }
                    if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC)) != 0)
                    {
                        string gns = " ";
                        if (gve.gnames != null)
                            foreach (string gs in gve.gnames)
                                gns += gs + " ";
                        Isis.WriteLine("Request GVUpdate: gve=(mode:" + gve.mode + ", request:" + rToString(gve.request) + ", who: " + gve.who + ", uid:" + gve.uid + ", gnames:{" + gns + "}, gaddrs:[" + Address.VectorToString(gve.gaddrs) + "]");
                    }
                    if (!ORACLE.TakingOver || (gve.request == LEAVE && ORACLE.theView.GetRawRankOf(gve.who) == ORACLE.theView.GetRawRankOf(Isis.my_address) - 1))
                    {
                        if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC)) != 0)
                            Isis.WriteLine("Calling RequestGVUpdates, gve list contains " + gveList.Count() + " gve items");
                        RequestGVUpdates(gveList);
                        gveList = new List<GVEvent>();
                        ORACLE.TakingOver = false;
                    }
                    bool sendIt = false;
                    using (new LockAndElevate(GVELock))
                        if (GVEList.Count() == 0 && Isis.ISISMEMBERS.HasFirstView)
                            sendIt = true;
                    if (sendIt)
                        // Although the system also has a way to do this on a per-group basis, the volume of P2P traffic it caused was excessive
                        Group.IPMCViewCast(Isis.ISISMEMBERS.theView.viewid, Isis.ISISMEMBERS.gaddr, Isis.my_address, Isis.ISISMEMBERS.theView);
                }
            }
            catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
            finally { OracleBeaconTaskWait.Release(1); }
            IsisSystem.ThreadTerminationMagic();
        }

        // Core of the virtual synchrony implementation, runs only in the leader.  The list will only contain one element unless
        // there is a new leader taking over in the ORACLE and it discovers one or more proposed events that it needs to order ahead
        // of the ORACLE leader FAIL proposal
        internal static void RequestGVUpdates(List<GVEvent> gveList)
        {
            int nCreate = 0;
            List<Group> AggLGWithNewOwner = new List<Group>(); // Large groups that will have a new group owner as a result of this GVEUpdate
            List<Group> AggCreateList = new List<Group>();
            List<Group> AggProposeGlist = new List<Group>();   // Don't participate in the 2PC (currently: regular groups being created by this action)
            List<Group> AggCommitGlist = new List<Group>();    // Vsync groups, they participate in the 2PC, as does the ORACLE
            List<Group> AggLargeGlist = new List<Group>();     // Large groups.  The ORACLE runs the 2PC and then unilaterally tells them to commit the new view

            int nProposed = 0, n = 0;

            List<ViewDelta> vdlist = new List<ViewDelta>();

            foreach (GVEvent gve in gveList)
                if (gve.request == JOIN)
                    for (int idx = 0; idx < gve.gnames.Length; idx++)
                    {
                        string gn = gve.gnames[idx];
                        if (!gn.Equals("ORACLE") && !gn.Equals("ISISMEMBERS"))
                        {
                            Group tpg = Group.TrackingProxyLookup(gn);
                            if (tpg != null && tpg.TypeSig != 0 && tpg.TypeSig != gve.tsigs[idx])
                            {
                                ReliableSender.SendPoison(gve.who, "TypeSignature mismatch in group <" + gn + ">");
                                Thread.Sleep(50);
                                Isis.NodeHasFailed(gve.who, "TypeSignature mismatch", false);
                                return;
                            }
                        }
                    }
            foreach (GVEvent gve in gveList)
            {
                List<Group> LGWithNewOwner = new List<Group>(); // Large groups that will have a new group owner as a result of this GVEUpdate
                List<Group> CreateList = new List<Group>();
                List<Group> ProposeGlist = new List<Group>();   // Don't participate in the 2PC (currently: regular groups being created by this action)
                List<Group> CommitGlist = new List<Group>();    // Vsync groups, they participate in the 2PC, as does the ORACLE
                List<Group> LargeGlist = new List<Group>();     // Large groups.  The ORACLE runs the 2PC and then unilaterally tells them to commit the new view
                if (gve.request == FDETECTION || gve.request == LEAVE)
                {
                    if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                        Isis.WriteLine("VUProtocol: FDETECTION/LEAVE event");
                    if (ORACLE.theView.GetRawRankOf(gve.who) != -1)
                    {
                        CommitGlist.Add(ORACLE);
                        ProposeGlist.Add(ORACLE);
                    }
                    using (new LockAndElevate(Group.TPGroupsLock))
                    {
                        foreach (var kvp in Group.TPGroups)
                        {
                            Group g = kvp.Value;
                            int r;
                            if ((gve.request == LEAVE && !gve.gnames.Contains(g.gname)) || (r = g.theView.GetRawRankOf(gve.who)) == -1)
                                continue;
                            if ((g.flags & Group.G_ISLARGE) != 0)
                            {
                                LargeGlist.Add(g);
                                if (r == 0)
                                    LGWithNewOwner.Add(g);
                            }
                            else if (CommitGlist.Contains(g) == false)
                            {
                                CommitGlist.Add(g);
                                if ((g.flags & Group.G_ISRAW) == 0)
                                    ProposeGlist.Add(g);
                            }
                        }
                    }
                }
                else
                {
                    if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC)) != 0)
                    {
                        string gs = " ", tss = " ", fs = " ";
                        foreach (string s in gve.gnames)
                            gs += s + " ";
                        if (gve.tsigs != null)
                            foreach (int ts in gve.tsigs)
                                tss += ts + " ";
                        if (gve.flags != null)
                            foreach (int fl in gve.flags)
                                fs += fl + " ";
                        Isis.WriteLine("VUProtocol: executing GVE event[" + gve.uid + "]: request " + Isis.rToString(gve.request) + " mode " + gve.mode + " address " + gve.who +
                            " gnames[" + gs + "] groups " + Address.VectorToString(gve.gaddrs) + "m ts={" + tss + "}, fs={" + fs + "}");
                    }
                    for (int i = 0; i < gve.gnames.Length; i++)
                    {
                        Group g;
                        if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC)) != 0)
                            Isis.WriteLine("VUProtocol: group name is " + gve.gnames[i]);
                        if (gve.gnames[i].Equals("ORACLE"))
                            g = ORACLE;
                        else
                        {
                            if ((g = Group.TrackingProxyLookup(gve.gaddrs[i])) == null || g.theView == null || g.theView.members.Length == 0)
                            {
                                if ((gve.mode & Group.CREATE) == 0)
                                {
                                    // Group.JOIN only but doesn't exist
                                    ORACLE.doP2PSend(gve.who, true, JOINFAILED, gve.gaddrs[i], "JoinExisting but group <" + gve.gnames[i] + "> didn't exist");
                                    return;
                                }
                                // Create tracking proxy for groups if needed
                                ++nCreate;
                                g = Group.TrackingProxy(gve.gnames[i], "VUP", gve.gaddrs[i], gve.tsigs[i], null, new View(gve.gnames[i], gve.gaddrs[i], new Address[] { gve.who }, -1, false), gve.flags[i], false);
                                CreateList.Add(g);
                                continue;
                            }
                            else if ((gve.mode & Group.JOIN) == 0)
                            {
                                // Group.JOIN only but doesn't exist
                                ORACLE.doP2PSend(gve.who, true, JOINFAILED, gve.gaddrs[i], "Create but group <" + gve.gnames[i] + "> already exists");
                                return;
                            }
                        }
                        if ((gve.flags[i] & Group.G_ISLARGE) != 0)
                            LargeGlist.Add(g);
                        else
                        {
                            CommitGlist.Add(g);
                            if (g.theView != null && g.theView.members.Length > 0 && (g.flags & Group.G_ISRAW) == 0)
                                ProposeGlist.Add(g);
                        }
                    }
                }

                if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                {
                    foreach (Group g in CreateList)
                        Isis.WriteLine("+++[+] Create GroupList contains <" + g.gname + ">");
                    foreach (Group g in ProposeGlist)
                        Isis.WriteLine("+++[P] Proposed GroupList contains <" + g.gname + ">"); ;
                    foreach (Group g in ProposeGlist)
                        Isis.WriteLine("+++[C] Commit GroupList contains <" + g.gname + ">");
                    foreach (Group g in LargeGlist)
                        Isis.WriteLine("+++[L] Large GroupList contains <" + g.gname + ">");
                }

                if (Isis.Proposed != null)
                    nProposed = Isis.Proposed.Length;
                else
                    nProposed = 0;

                Address[] wantJoin = null, wantLeave = null;
                if (gve.request == JOIN)
                {
                    bool fnd = false;
                    foreach (ViewDelta vd in vdlist)
                        if (vd.leavers.Length != 0 || vd.leaderId != LeaderId)
                            continue;
                        else
                            foreach (Address ga in gve.gaddrs)
                                if (ga.Equals(vd.gaddr))
                                {
                                    fnd = true;
                                    Isis.ArrayResize<Address>(ref vd.joiners, vd.joiners.Length + 1);
                                    vd.joiners[vd.joiners.Length - 1] = gve.who;
                                }
                    if (!fnd)
                    {
                        foreach (ViewDelta vd in vdlist)
                            if (gve.gaddrs.Contains(vd.gaddr) && vd.joiners.Contains(gve.who))
                            {
                                fnd = true;
                                break;
                            }
                        if (!fnd)
                        {
                            wantJoin = new Address[] { gve.who };
                            wantLeave = new Address[0];
                        }
                    }
                }
                else
                {
                    bool fnd = false;
                    foreach (ViewDelta vd in vdlist)
                        if (vd.joiners.Length != 0 || vd.leaderId != LeaderId)
                            continue;
                        else
                            foreach (Address ga in gve.gaddrs)
                                if (ga.Equals(vd.gaddr))
                                {
                                    fnd = true;
                                    Isis.ArrayResize<Address>(ref vd.leavers, vd.leavers.Length + 1);
                                    vd.leavers[vd.leavers.Length - 1] = gve.who;
                                }
                    if (!fnd)
                    {
                        foreach (ViewDelta vd in vdlist)
                            if (gve.gaddrs.Contains(vd.gaddr) && vd.leavers.Contains(gve.who))
                            {
                                fnd = true;
                                break;
                            }
                        if (!fnd)
                        {
                            wantLeave = new Address[] { gve.who };
                            wantJoin = new Address[0];
                        }
                    }
                }

                // If ORACLE group has a failure that will take it below ISIS_ORACLESIZE, see if we have any available candidates
                // that could replace the departing members(s) and if so, add them to a list of proposed joiners.
                if (gve.gaddrs.Length == 1 && gve.gaddrs[0].Equals(ORACLE.gaddr) && wantLeave.Length > 0)
                {
                    using (new LockAndElevate(CanBeOracleListLock))
                    {
                        while (CanBeOracleList.Count() > 0)
                        {
                            Address a = CanBeOracleList.First();
                            CanBeOracleList.Remove(a);
                            if (ORACLE.GetRankOf(a) != -1)
                                continue;
                            wantJoin = new Address[] { a };
                            break;
                        }
                    }
                }

                if (nProposed > 0)
                {
                    // Pending stuff I know (or learned) about
                    foreach (ViewDelta vd in Isis.Proposed)
                        vdlist.Add(vd);
                }
                Isis.Proposed = null;

                if (wantJoin != null && wantLeave != null)
                {
                    foreach (Group g in CommitGlist)
                        AddVD(vdlist, wantJoin, wantLeave, g, gve.offset);
                    foreach (Group g in LargeGlist)
                        AddVD(vdlist, wantJoin, wantLeave, g, gve.offset);
                    foreach (Group g in CreateList)
                        AddVD(vdlist, wantJoin, wantLeave, g, gve.offset);
                }

                // Now sweep everything into the corresponding aggregated lists and repeat if there are more GVE entries, which happens only if
                // a new leader is taking over and needs to include one or more pending proposals with the LEAVE for the old ORACLE leader
                AddUnique(AggLGWithNewOwner, LGWithNewOwner);
                AddUnique(AggCreateList, CreateList);
                AddUnique(AggProposeGlist, ProposeGlist);
                AddUnique(AggCommitGlist, CommitGlist);
                AddUnique(AggLargeGlist, LargeGlist);
            }

            if (AggCommitGlist.Contains(ORACLE) == false)
            {
                AggProposeGlist.Add(ORACLE);
                AggCommitGlist.Add(ORACLE);
            }

            if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC)) != 0)
                Isis.WriteLine("VUProtocol: Lists [ " + AggCommitGlist.Count() + " commit, " + AggProposeGlist.Count() + " propose, " + AggLargeGlist.Count() + " Large(SameOwner), " + AggLGWithNewOwner.Count() + " Large(NewOwner) " + AggCreateList.Count() + " create]");

            ViewDelta[] vds = vdlist.ToArray();

            if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC)) != 0)
            {
                Isis.WriteLine("VUProtocol: About to Propose View Deltas:");
                foreach (ViewDelta vd in vds)
                    Isis.WriteLine("     " + vd);
            }

            List<Address> whoFailed = new List<Address>();

            // For each group in glist, we'll get a reply from each member, in the form of a byte-vector
            List<byte[]>[] ba = null;

            // This will initialize whoFailed with the list of processes shown as failed 
            // for groups in glist.  No point in expecting them to reply to the PROPOSE solicitation
            noteFailed(AggProposeGlist, whoFailed);
            bool must_loop = true;
            UnstableList[] usl = new UnstableList[0];
            int oldLen = 0, nreplies = 0;
            int loopLimit = CountLive(AggProposeGlist);
            int nExpected = loopLimit;
            while (must_loop)
            {
                if (loopLimit-- < 0)
                {
                    string gstr = " ", vdStr = "", rstr = " ";
                    foreach (Group g in AggProposeGlist)
                        gstr += g.gname + " ";
                    foreach (ViewDelta vd in vds)
                        vdStr += ">> " + vd + "\r\n";
                    for (int gn = 0; gn < ba.Length; gn++)
                        rstr += ba[gn].Count() + " ";
                    throw new IsisException("Trapped looping in PROPOSE to <" + gstr + ">! (nreplies " + nreplies + ", nExpected " + nExpected + " (" + rstr + "), usl.Length " + usl.Length +
                        ", oldLen " + oldLen + ")\r\nViewDeltas:\r\n" + vdStr + IsisSystem.GetState());
                }
                must_loop = false;
                oldLen = usl.Length;
                if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                {
                    string dests = " ";
                    foreach (Group g in AggProposeGlist)
                        dests += g.gname + " ";
                    Isis.WriteLine("Sending the PROPOSE messages to [" + dests + "]");
                }
                // Flush can take a long time, so disable the timeout
                ba = Group.doMultiQuery(AggProposeGlist, Group.ALL, true, new Timeout(Int32.MaxValue, Timeout.TO_FAILURE, "PROPOSE"), PROPOSE, vds, usl);
                nreplies = 0;
                for (int gn = 0; gn < ba.Length; gn++)
                    nreplies += ba[gn].Count();
                if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                    Isis.WriteLine("After multiquery tabulate results from " + ba.Length + " groups that were queried (total of " + nreplies + " distinct replies");
                if (Tabulate(AggProposeGlist, ba, vds, ref usl) < (ORACLE.theView.members.Length + 1) / 2 && ISIS_IGNOREPARTITIONS == false)
                    throw new IsisException("ISIS experienced a loss of majority, terminating.");
                if (nreplies != nExpected || usl.Length > oldLen)
                {
                    int newExpected = CountLive(AggProposeGlist);
                    if (newExpected == nExpected)
                        break;
                    must_loop = true;
                    nExpected = newExpected;
                }
                if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                    Isis.WriteLine("After tabulate must_loop " + must_loop + "(nreplies " + nreplies + ", nExpected " + nExpected + "; usl.Length " + usl.Length + ", oldLen " + oldLen + ")");
            }
            if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
            {
                Isis.Write("Sending the COMMIT messages to ");
                foreach (Group g in AggCommitGlist)
                    Isis.Write("<" + g.gname + ">");
                Isis.WriteLine();
            }
            Address[] whos = new Address[gveList.Count()];
            int[] uids = new int[gveList.Count()];
            n = 0;
            foreach (GVEvent gve in gveList)
            {
                whos[n] = gve.who;
                uids[n] = gve.uid;
                ++n;
            }

            foreach (ViewDelta vd in vds)
            {
                foreach (Address who in Expand(vd.leavers))
                {
                    Group g = Group.TrackingProxyLookup(vd.gaddr);
                    if (g != null && g.theView != null)
                        ReliableSender.SendP2P(Msg.ISGRPP2P, who, g, g.theView.viewid, ReliableSender.P2PSequencer.NextP2PSeqn("commit", who),
                            Msg.toBArray(COMMIT, vds, whos, uids), true, null, null);
                }
            }

            // Now perform the COMMIT operations, first for groups other than the ORACLE, and then for the ORACLE itself
            List<Address> vdsg = new List<Address>();
            foreach (ViewDelta vd in vds)
                if (!vdsg.Contains(vd.gaddr) && !Isis.ORACLE.gaddr.Equals(vd.gaddr))
                    vdsg.Add(vd.gaddr);
            if (vdsg.Count() > 0)
            {
                Group.doMultiSend(AggCommitGlist.Where(g => vdsg.Contains(g.gaddr)).ToList(), true, COMMIT, vds, whos, uids);
                foreach (Group g in AggCommitGlist)
                    if (g != Isis.ORACLE)
                        g.Flush(Isis.ISIS_ACKTHRESHOLD);
            }
            Isis.ORACLE.doSend(true, false, COMMIT, vds, whos, uids);

            if (AggLargeGlist.Count() > 0)
                foreach (Group g in AggLargeGlist)
                    sendVDS(g, vds, AggLGWithNewOwner.Contains(g));

            if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                Isis.WriteLine("Calling CommitWait");
            ILock.Barrier(ILock.LLWAIT, ILock.LCOMMIT).BarrierWait();
            if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                Isis.WriteLine("After CommitWait");
        }


        // This is actually called from the ORACLE commit logic and is called in each ORACLE member.  As a result we redundantly send
        // INITIALVIEW messages to the group members, but the advantage is that the scheme is tolerant of a crash of the ORACLE leader
        // during a group join.  Obviously one could clean this up using logic similar to the DoAsLeader scheme, but because the INITIALVIEW
        // is being sent to a process not currently in the group, there is no simple way to get around asking that joining process if it
        // has the INITIALVIEW yet or not.  That gets a bit tricky (given the asynchrony of the failure notification that breaks the connection
        // from the old leader to the joining member).  So we're paying a price in extra messages, which get ignored, and the benefit is
        // simpler code (assuming you call this simpler... you may disagree!)
        internal static void SendInitialView(ViewDelta[] vds)
        {
            if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
            {
                Isis.WriteLine("SendInitialView.... view deltas:");
                foreach (ViewDelta vd in vds)
                    Isis.WriteLine("++++++ " + vd);
            }
            foreach (ViewDelta vd in vds)
            {
                Address[] joiners = Expand(vd.joiners);
                if (vd.gaddr.Equals(ORACLE.gaddr))
                {
                    foreach (Address who in joiners)
                        SendOracleInitialView(vd, who);
                }
                else
                {
                    Group g = Group.TrackingProxyLookup(vd.gaddr);
                    View theView;
                    if (g == null)
                        throw new IsisException("Can't find the group in VUCommit");
                    using (new LockAndElevate(g.ViewLock))
                        theView = g.theView;
                    if (theView == null)
                        throw new IsisException("View is null for " + g.gname + " in VUCommit: ViewDelta=" + vd + IsisSystem.GetState());
                    if (Isis.ISISMEMBERS != null && theView.members.Length >= Isis.ISIS_ORACLESIZE && !joiners.Contains(Isis.my_address) && (joiners.Length > Isis.ISIS_INITVIAOOB / 10 || theView.members.Length > Isis.ISIS_INITVIAOOB))
                        SendViewViaOOB(g, theView, joiners, vds, vd);
                    else foreach (Address who in joiners)
                            SendNonOracleInitialView(vds, vd, who);
                }
            }
        }

        private static void SendOracleInitialView(ViewDelta vd, Address newOracleMember)
        {
            string[] names;
            Address[] gaddrs;
            long[] tsigs;
            View[] vs;
            bool[] isl;
            int[,] mms;
            if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                Isis.WriteLine("+++ newOracleMember true.");
            int n;
            using (new LockAndElevate(Group.TPGroupsLock))
            {
                int ng = Group.TPGroups.Count() + 1;
                foreach (var kvp in Group.TPGroups)
                    if (kvp.Value.HasFirstView == false)
                        --ng;
                names = new string[ng];
                gaddrs = new Address[ng];
                tsigs = new long[ng];
                vs = new View[ng];
                isl = new bool[ng];
                n = 0;
                foreach (var kvp in Group.TPGroups)
                    if (kvp.Value.HasFirstView)
                        includeGroup(n++, names, gaddrs, tsigs, vs, isl, kvp.Value);
            }
            includeGroup(n, names, gaddrs, tsigs, vs, isl, Isis.ORACLE);
            MCMDSocket.SetMap("ORACLE:SendOracleInitialView", "ORACLE", false, MCMDSocket.GetMap(Isis.ORACLE.gaddr, true));
            // This sends the list of groups and tracking proxies to a new oracle member
            // It also includes the view of the oracle itself
            if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC | IsisSystem.STARTSEQ)) != 0)
                Isis.WriteLine("ORACLE.P2PSend INITALVIEW for " + Address.VectorToString(gaddrs) + " from " + Isis.my_address + " to " + newOracleMember + " MCMD STATE + " + MCMDSocket.GetState());
            mms = MCMDSocket.GetMap(gaddrs);
            Address[] CBMs;
            using (new LockAndElevate(Isis.CanBeOracleListLock))
                CBMs = Isis.CanBeOracleList.ToArray();
            ORACLE.doP2PSend(newOracleMember, true, INITIALVIEW, Isis.my_address, names, gaddrs, tsigs, vs, isl, MCMDSocket.nextPhysIPAddr, mms, CBMs);
        }

        private static void SendNonOracleInitialView(ViewDelta[] vds, ViewDelta vd, Address newGroupMember)
        {
            Group g = Group.TrackingProxyLookup(vd.gaddr);
            if (g == null)
                throw new IsisException("Can't find the group in VUCommit");
            if (g.theView == null)
                throw new IsisException("View is null for " + g.gname + " in VUCommit: ViewDelta=" + vd + IsisSystem.GetState());
            if ((g.flags & Group.G_ISLARGE) == 0 || vd.prevVid == -1)
            {
                if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                    Isis.WriteLine("Group <" + g.gname + ">, sending INITIALVIEW to " + newGroupMember + ", VIEW = " + g.theView);
                ReliableSender.SendP2P(Msg.ISGRPP2P, newGroupMember, g, g.theView.viewid, ReliableSender.P2PSequencer.NextP2PSeqn("intialview", newGroupMember),
                    Msg.toBArray(INITIALVIEW, Isis.my_address, g.theView, MCMDSocket.GetMap(g.gaddr, true), 0), true, null, null);
            }
            if ((g.flags & Group.G_ISLARGE) != 0)
                Isis.sendVDS(g, vds, vd.joiners.Length > 0);
        }

        private static void SendViewViaOOB(Group g, View theView, Address[] joiners, ViewDelta[] vds, ViewDelta vd)
        {
            if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                Isis.WriteLine("Group <" + g.gname + ">, sending INITIALVIEW via OOB to " + Address.VectorToString(joiners) + ", VIEW = " + g.theView);
            byte[] ba = theView.toBArray();
            string fname = g.gname;
            int lio = fname.LastIndexOfAny(new char[] { '\\', '/' });
            if (lio != -1)
                fname = fname.Substring(lio + 1);
            fname += "-view" + theView.viewid;
            MemoryMappedFile mmf;
            try
            {
                mmf = MemoryMappedFile.CreateNew(fname, ba.Length);
                MemoryMappedViewAccessor mva = mmf.CreateViewAccessor();
                mva.WriteArray<byte>(0, ba, 0, ba.Length);
            }
            catch (IOException)
            {
                mmf = MemoryMappedFile.OpenExisting(fname);
            }
            Isis.ISISMEMBERS.OOBRegister(true, fname, Isis.ISISMEMBERS.gaddr, mmf, Isis.ORACLE.theView.members.ToList<Address>());
            List<Address> replicas = new List<Address>() { Isis.my_address };
            foreach (Address j in joiners)
                replicas.Add(j);
            Isis.ISISMEMBERS.OOBReReplicate(true, new List<Group.OOBRepInfo>() { new Group.OOBRepInfo(fname, Isis.ISISMEMBERS.gaddr, replicas) },
                (Action<string, MemoryMappedFile>)delegate(string oobfname, MemoryMappedFile mf) { g.OOBDelete(oobfname); });
            foreach (Address newGroupMember in joiners)
                ReliableSender.SendP2P(Msg.ISGRPP2P, newGroupMember, g, theView.viewid, ReliableSender.P2PSequencer.NextP2PSeqn("intialview", newGroupMember),
                    Msg.toBArray(INITIALVIEW, Isis.my_address, fname, MCMDSocket.GetMap(g.gaddr, true), 0), true, null, null);
            if ((g.flags & Group.G_ISLARGE) != 0)
                Isis.sendVDS(g, vds, vd.joiners.Length > 0);
        }

        private static void AddUnique(List<Group> AggregatedList, List<Group> GroupsToAdd)
        {
            foreach (Group g in GroupsToAdd)
                if (!AggregatedList.Contains(g))
                    AggregatedList.Add(g);
        }

        private static void AddVD(List<ViewDelta> vdlist, Address[] wantJoin, Address[] wantLeave, Group g, long offset)
        {
            View theView;
            using (new LockAndElevate(g.ViewLock))
                theView = g.theView;
            List<Address> prevMembers = (theView == null) ? new List<Address>() : prevMembers = theView.members.ToList();
            int prevVid = -1;
            foreach (ViewDelta vd in vdlist)
            {
                foreach (Address a in Expand(vd.leavers))
                    prevMembers.Remove(a);
                foreach (Address a in Expand(vd.joiners))
                    prevMembers.Add(a);
                if (prevMembers.Count() == 0)
                    prevVid = -1;
                else if (vd.gaddr.Equals(g.gaddr) && vd.prevVid >= prevVid)
                    prevVid = vd.prevVid + 1;
            }
            if (prevVid == -1)
                prevVid = g.theView.viewid;
            vdlist.Add(new ViewDelta(g.gname, g.gaddr, g.TypeSig, g.MCMDMAP(), prevVid, g.theView.members.Length, wantJoin, offset, wantLeave, (g.flags & Group.G_ISLARGE) != 0));
        }

        /*
         * Large group: Send the owner (only) the view deltas for that group 
         * If the new owner isn't the same as the previous owner, however, also send
         * previous view deltas that he might have missed (e.g. not yet known to be stable)
         */
        private static LockObject oldVDSlock = new LockObject("oldVDSLock");
        private static ViewDelta[] oldVDS;
        private static long[] vdTimes;

        internal static void clearOldVDS(Address gaddr, int StableTo)
        {
            using (new LockAndElevate(oldVDSlock))
            {
                // First count the ones to keep
                int cnt = 0;
                if (oldVDS != null)
                {
                    for (int i = 0; i < oldVDS.Length; i++)
                        if (oldVDS[i].gaddr.Equals(gaddr) && oldVDS[i].prevVid > StableTo)
                            ++cnt;
                }
                ViewDelta[] newVDS = new ViewDelta[cnt];
                if (oldVDS != null)
                {
                    cnt = 0;
                    for (int i = 0; i < oldVDS.Length; i++)
                        if (oldVDS[i].gaddr.Equals(gaddr) && oldVDS[i].prevVid > StableTo)
                            newVDS[cnt++] = oldVDS[i];
                }
                oldVDS = newVDS;
            }
        }

        private static void sendVDS(Group g, ViewDelta[] vds, bool includePrev)
        {
            ViewDelta[] vdsToApply;
            long[] times;
            int cnt = 0;
            using (new LockAndElevate(oldVDSlock))
            {
                // Count the number of ViewDeltas to send.  First any lingering old ones
                if (includePrev && oldVDS != null)
                    cnt = oldVDS.Length;
                foreach (ViewDelta vd in vds)
                    if (vd.gaddr.Equals(g.gaddr))
                        ++cnt;
                if (cnt == 0)
                    return;
                vdsToApply = new ViewDelta[cnt];
                times = new long[cnt];
                cnt = 0;
                if (includePrev && oldVDS != null)
                    for (int i = 0; i < oldVDS.Length; i++)
                        if ((Isis.NOW() - vdTimes[i]) < 180000L && !vdDup(vdsToApply, cnt, oldVDS[i]))  // 3 minutes
                        {
                            times[cnt] = vdTimes[cnt];
                            vdsToApply[cnt++] = oldVDS[i];
                        }
                for (int i = 0; i < vds.Length; i++)
                    if (vds[i].gaddr.Equals(g.gaddr) && !vdDup(vdsToApply, cnt, vds[i]))
                    {
                        times[cnt] = Isis.NOW();
                        vdsToApply[cnt++] = vds[i];
                    }
                if (cnt < vdsToApply.Length)
                    Isis.ArrayResize<ViewDelta>(ref vdsToApply, cnt);
                oldVDS = vdsToApply;
                vdTimes = times;
            }
            g.gotNewViewDeltas(vdsToApply);
            int ownerRank = 0;
            while (ownerRank < g.theView.members.Length && (g.theView.hasFailed[ownerRank] || isLeaving(g.theView.members[ownerRank], vdsToApply)))
                ++ownerRank;
            if (ownerRank == g.theView.members.Length)
            {
                if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                    Isis.WriteLine("SendVDS discovered that a large group experienced a total failure; terminating it");
                g.Terminate();
                return;
            }
            Address sendTo = g.theView.members[ownerRank];
            if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
            {
                Isis.WriteLine("Sending LgCOMMIT via RELAYSEND to LgOwner " + g.theView.members[ownerRank] + " for view deltas:");
                foreach (Isis.ViewDelta vd in vdsToApply)
                    Isis.WriteLine("   " + vd);
            }
            g.doP2PSend(sendTo, true, RELAYSEND, vdsToApply);
        }

        private static bool vdDup(ViewDelta[] vds, int cnt, ViewDelta nvd)
        {
            ViewDelta vd;
            if (nvd == null)
                return true;
            for (int n = 0; n < cnt; n++)
            {
                if ((vd = vds[n]) == null)
                    break;
                if (vd.gaddr.Equals(nvd.gaddr) && vd.leaderId == nvd.leaderId && vd.prevVid == nvd.prevVid)
                    return true;
            }
            return false;
        }

        private static bool isLeaving(Address who, Isis.ViewDelta[] vds)
        {
            foreach (Isis.ViewDelta vd in vds)
                foreach (Address l in vd.leavers)
                    if (l.Equals(who))
                        return true;
            return false;
        }

        private static void includeGroup(int n, string[] names, Address[] gaddrs, long[] tsigs, View[] vs, bool[] isl, Group g)
        {
            names[n] = g.gname;
            gaddrs[n] = g.gaddr;
            tsigs[n] = g.TypeSig;
            vs[n] = g.theView;
            isl[n++] = (g.flags & Group.G_ISLARGE) != 0;
            if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                Isis.WriteLine("Including initial view info: <" + g.gname + ">, <" + g.gaddr + ">, view " + g.theView);
        }

        private static int CountLive(List<Group> glist)
        {
            int nLive = 0;
            foreach (Group g in glist)
                if (g.theView == null)
                    throw new IsisException("theView null for group " + g.gname + " in CountLive");
                else
                    for (int m = 0; m < g.theView.members.Length; m++)
                        if (g.theView.hasFailed[m] == false)
                            using (new LockAndElevate(Isis.RIPLock))
                                if (!Isis.RIPList.Contains(g.theView.members[m]))
                                {
                                    if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                                        Isis.WriteLine("PROPOSE/CountLive: <" + g.gname + "> expecting a reply from " + g.theView.members[m]);
                                    ++nLive;
                                }
                                else if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                                    Isis.WriteLine("PROPOSE/CountLive: <" + g.gname + "> NOT expecting a reply from " + g.theView.members[m] + " (shown as dead in this view)");
            return nLive;
        }

        private static List<Address> noteFailed(List<Group> glist, List<Address> whoFailed)
        {
            List<Address> newFailures = new List<Address>();
            foreach (Group g in glist)
                if (g.theView == null)
                    throw new IsisException("theView null for group " + g.gname + " in noteFailed");
                else
                    for (int m = 0; m < g.theView.members.Length; m++)
                        if (g.theView.hasFailed[m] && whoFailed.Contains(g.theView.members[m]) == false)
                        {
                            whoFailed.Add(g.theView.members[m]);
                            newFailures.Add(g.theView.members[m]);
                        }
            return newFailures;
        }

        delegate void tabulator(Address who, ViewDelta[] rvds);

        // Updates vds to list terminal message send counts, computes the flush vector,
        //  and then returns the number of Oracle members who replied to the proposal
        private static int Tabulate(List<Group> glist, List<byte[]>[] bas, ViewDelta[] vds, ref UnstableList[] usl)
        {
            int gn, ngroups = glist.Count();
            List<Address> GotResponseFrom = new List<Address>();
            Group[] garray = glist.ToArray();
            if (bas.Length != ngroups)
                throw new IsisException("Inconsistency in Tabulate: glist.Count() was " + ngroups + ", but reply byte array contained " + bas.Length + " reply vectors!");
            UnstableList[][] UnstableMsgs = new UnstableList[bas.Length][];
            for (gn = 0; gn < ngroups; gn++)
            {
                Group g = garray[gn];
                using (new LockAndElevate(g.ViewLock))
                {
                    UnstableMsgs[gn] = new UnstableList[g.theView.members.Length];
                    for (int who = 0; who < g.theView.members.Length; who++)
                        UnstableMsgs[gn][who] = new UnstableList(g.gaddr, NULLADDRESS, g.theView.members[who], g.theView.viewid, -1, -1);
                }
            }
            int nOracleReplies = 0;
            for (int gba = 0; gba < bas.Length; gba++)
            {
                List<byte[]> baGroup = bas[gba];
                for (int outerWho = 0; outerWho < baGroup.Count(); outerWho++)
                {
                    int who = outerWho;
                    byte[] ba = baGroup[who];
                    object[] obs = Msg.BArrayToObjects(ba);
                    // Msg.InvokeFromBArray(ba, (tabulator)delegate(Address sender, ViewDelta[] rvds)
                    if (obs.Length == 2 && obs[0].GetType().Equals(typeof(Address)) && obs[1].GetType().Equals(typeof(ViewDelta[])))
                    {
                        Address sender = (Address)obs[0];
                        GotResponseFrom.Add(sender);
                        ViewDelta[] rvds = (ViewDelta[])obs[1];
                        if (rvds.Length == 0)
                            ++nOracleReplies;
                        else
                        {
                            bool oFound = false;
                            for (int n = 0; n < vds.Length; n++)
                            {
                                ViewDelta rvd = rvds[n];
                                Group g;

                                if (rvd.gaddr.Equals(ORACLE.gaddr))
                                    g = ORACLE;
                                else if ((g = Group.TrackingProxyLookup(rvd.gaddr)) == null)
                                {
                                    // Very rarely, I've seen this happen and am unclear how the race arises... 
                                    Thread.Sleep(2500);
                                    if ((g = Group.TrackingProxyLookup(rvd.gaddr)) == null)
                                        return 0;
                                }

                                if (!oFound && g == ORACLE)
                                {
                                    oFound = true;
                                    ++nOracleReplies;
                                }

                                if (rvd.prevVid == vds[n].prevVid)
                                {
                                    gn = -1;
                                    for (int i = 0; i < vds[n].lastSeqns.Length; i++)
                                        if (rvd.lastSeqns.Length == 0)
                                            continue;
                                        else if (vds[n].lastSeqns[i] != rvd.lastSeqns[i])
                                        {
                                            if (gn == -1)
                                            {
                                                for (gn = 0; gn < ngroups; gn++)
                                                    if (glist[gn].gaddr.Equals(rvd.gaddr))
                                                        break;
                                                if (gn == ngroups)
                                                    throw new IsisException("Inconsistency in Tabulate: glist didn't contain group " + rvd.gaddr);
                                            }
                                            if (vds[n].lastSeqns[i] < rvd.lastSeqns[i])
                                            {
                                                UnstableMsgs[gn][i].flusher = sender;
                                                if (UnstableMsgs[gn][i].mid_low == -1)
                                                    UnstableMsgs[gn][i].mid_low = rvd.lastSeqns[i];
                                                else if (UnstableMsgs[gn][i].mid_hi != -1)
                                                    UnstableMsgs[gn][i].mid_low = Math.Min(UnstableMsgs[gn][i].mid_low, UnstableMsgs[gn][i].mid_hi);
                                                UnstableMsgs[gn][i].mid_hi = rvd.lastSeqns[i];
                                                vds[n].lastSeqns[i] = rvd.lastSeqns[i];
                                            }
                                            else
                                                UnstableMsgs[gn][i].mid_low = rvd.lastSeqns[i];
                                        }
                                }
                            }
                        }
                    }
                    else
                        throw new IsisException("in Tabulate expected reply signature { Isis.Address  Isis.Isis+ViewDelta[] }");
                }
            }
            int nUnstable = 0;
            for (gn = 0; gn < ngroups; gn++)
                for (int who = 0; who < UnstableMsgs[gn].Length; who++)
                    if (!GotResponseFrom.Contains(UnstableMsgs[gn][who].sender) && UnstableMsgs[gn][who].mid_hi != -1 && UnstableMsgs[gn][who].mid_low != -1)
                        nUnstable++;
            usl = new UnstableList[nUnstable];
            nUnstable = 0;
            for (gn = 0; gn < ngroups; gn++)
                for (int who = 0; who < UnstableMsgs[gn].Length; who++)
                    if (!GotResponseFrom.Contains(UnstableMsgs[gn][who].sender) && UnstableMsgs[gn][who].mid_hi != -1 && UnstableMsgs[gn][who].mid_low != -1)
                        usl[nUnstable++] = UnstableMsgs[gn][who];
            return nOracleReplies;
        }

        // Runs in all participants
        internal static void CommitGVUpdates(Group g, ViewDelta[] vds)
        {
            View v = null;
            CommitGVUpdates(g, vds, ref v);
        }

        internal static void CommitGVUpdates(Group g, ViewDelta[] vds, ref View newView)
        {
            bool IamOracle = (g == ORACLE && g.theView.GetMyRank() != -1);
            if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC)) != 0)
            {
                string vs = " ";
                foreach (ViewDelta vd in vds)
                    vs += "    " + vd + "\r\n";
                Isis.Write("CommitViewUpdates<" + g.gname + ">... IAmOracle=" + IAmOracle + ":\r\n" + vs);
            }
            long before = Isis.NOW();
            if (IamOracle)
            {
                List<ViewDelta> vdsApplied = new List<ViewDelta>();
                foreach (ViewDelta vd in vds)
                {
                    if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC)) != 0)
                        Isis.WriteLine("... CommitGVUpdates(IAmOracle) now applying " + vd);
                    if (!vd.gaddr.Equals(ORACLE.gaddr))
                    {
                        Group tpg = Group.TrackingProxyLookup(vd.gaddr);
                        if (tpg == null)
                            tpg = Group.TrackingProxy(vd.gname, "Commit GVUPdates", vd.gaddr, vd.tsig, vd.mcmdmap, new View(vd.gname, vd.gaddr, vd.joiners, vd.prevVid + 1, false), vd.isLarge ? Group.G_ISLARGE : 0, false);
                        UpdateGroupView(true, vd, tpg, "TrackingProxy", ref newView);
                    }
                    else
                        UpdateGroupView(true, vd, g, "ORACLE:self-update", ref newView);
                    vdsApplied.Add(vd);
                }
                using (new LockAndElevate(GVELock))
                {
                    List<GVEvent> newGVEList = new List<GVEvent>();
                    foreach (GVEvent gve in GVEList)
                    {
                        bool fnd_all = true;
                        foreach (Address gaddr in gve.gaddrs)
                        {
                            bool fnd = false;
                            foreach (ViewDelta vd in vdsApplied)
                                if (vd.gaddr.Equals(gaddr) && (gve.request == JOIN ? vd.joiners.Contains(gve.who) : vd.leavers.Contains(gve.who)))
                                {
                                    fnd = true;
                                    break;
                                }
                            if ((fnd_all &= fnd) == false)
                                break;
                        }
                        if (fnd_all == false)
                            newGVEList.Add(gve);
                    }
                    GVEList = newGVEList;
                }
            }
            else
            {
                foreach (ViewDelta vd in vds)
                    if (vd.gaddr.Equals(g.gaddr))
                    {
                        if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.GVELOGIC)) != 0)
                            Isis.WriteLine("... CommitGVUpdates now applying " + vd);
                        UpdateGroupView(true, vd, g, "IsisGroups", ref newView);
                    }
            }
            if ((IsisSystem.Debug & IsisSystem.DELAYS) != 0 && (Isis.NOW() - before) > Isis.ISIS_WARNAFTER)
                Isis.WriteLine("WARNING: LONG DELAY while calling UpdateGroupView (" + (Isis.NOW() - before) + "ms)");
        }

        // Called with GroupIsReal from CommitGVUpdates
        // Called with GroupIsReal false when computing a "working view" in the token tree manager
        internal static void UpdateGroupView(bool GroupIsReal, ViewDelta vd, Group g, string queue)
        {
            View newView = null;
            UpdateGroupView(GroupIsReal, vd, g, queue, ref newView, false);
        }

        internal static void UpdateGroupView(bool GroupIsReal, ViewDelta vd, Group g, string queue, ref View nv)
        {
            UpdateGroupView(GroupIsReal, vd, g, queue, ref nv, false);
        }

        // Note that inhibitActions is only used with GroupIsReal set to false...
        internal static void UpdateGroupView(bool GroupIsReal, ViewDelta vd, Group g, string queue, ref View nv, bool inhibitActions)
        {
            if (g != Isis.ORACLE && !vd.gaddr.Equals(g.gaddr))
                return;
            if (GroupIsReal)
                using (new LockAndElevate(g.ViewLock))
                    nv = g.theView;
            Address[] joiners = Expand(vd.joiners);
            Address[] leavers = Expand(vd.leavers);
            if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                Isis.WriteLine("UpdateGroupView<" + g.gname + (GroupIsReal ? "" : " TrackingProxy") + ">: viewdelta= " + vd + ", expanded joiners = " + Address.VectorToString(joiners) + "; leavers " + Address.VectorToString(leavers));
            bool IAmLeaving = false;
            List<Address> newView = new List<Address>();
            if (g.theView == null)
                throw new IsisException("g.theView null in UpdateGroupView");
            if (GroupIsReal)
            {
                if (g.theView.viewid > vd.prevVid)
                {
                    if (vd.prevVid != -1 && (IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                        Isis.WriteLine("WARNING: UpdateGroupView ignoring a superfluous view update because <" + g.gname + "> is on view " + g.theView.viewid + ", but vd.prevVid is " + vd.prevVid + " in state " + IsisSystem.GetState());
                    return;
                }
                if (g.theView.viewid != vd.prevVid)
                    if ((g.flags & Group.G_ISLARGE) == 0 || g.theView.viewid <= vd.prevVid)
                    {
                        if (g.theView.viewid > 0)  // Viewid can "reset" if all members leave; this case arises if a concurrent join was underway...
                            Isis.WriteLine("WARNING... in CommitGVUpdates: <" + g.gname + ">: updating wrong view! (Expected " + vd.prevVid
                                + ", found " + g.theView.viewid + ")... group was on " + queue + "\r\nView Delta:" + vd + "\r\n");
                        g.GotLastSeqns(vd.prevVid, new int[g.theView.members.Length]);
                    }
                    else
                        return;
                else
                    g.GotLastSeqns(vd.prevVid, vd.lastSeqns);
            }
            int lcnt = 0, jcnt = 0, ntotal = g.theView.members.Length;
            Address[] validatedLeavers = new Address[leavers.Length];
            Address[] validatedJoiners = new Address[joiners.Length + ((g.myTargetGroupSize > 0 && ntotal > g.myTargetGroupSize) ? leavers.Length : 0)];
            for (int i = 0; i < ntotal; i++)
            {
                Address a = g.theView.members[i];
                if (leavers.Contains(a))
                {
                    validatedLeavers[lcnt++] = a;
                    if (a.isMyAddress())
                        IAmLeaving = true;
                    if (g.myTargetGroupSize > 0 && ntotal > g.myTargetGroupSize)
                    {
                        while (ntotal > i && leavers.Contains(g.theView.members[ntotal - 1]))
                            --ntotal;
                        if (ntotal > g.myTargetGroupSize)
                        {
                            Address who = g.theView.members[--ntotal];
                            newView.Add(who);
                            validatedJoiners[jcnt++] = who;
                        }
                    }
                }
                else
                    newView.Add(a);

            }
            foreach (Address a in joiners)
                if (!newView.Contains(a))
                {
                    newView.Add(a);
                    validatedJoiners[jcnt++] = a;
                }
            if (vd.prevVid == -1 && !validatedJoiners.Contains(Isis.my_address))
                validatedJoiners[jcnt++] = Isis.my_address;
            if (lcnt != validatedLeavers.Length)
                Isis.ArrayResize<Address>(ref validatedLeavers, lcnt);
            if (jcnt != validatedJoiners.Length)
                Isis.ArrayResize<Address>(ref validatedJoiners, jcnt);
            if (newView.Count() == 0 && g.isTrackingProxy)
            {
                g.GroupClose();
                using (new LockAndElevate(Group.TPGroupsLock))
                    Group.TPGroups.Remove(g.gaddr);
                return;
            }
            View nextView = new View(vd.gname, vd.gaddr, newView.ToArray(), vd.prevVid + 1, (g.flags & Group.G_ISLARGE) != 0);
            if (g.isTrackingProxy == false)
            {
                nextView.joiners = validatedJoiners;
                nextView.offset = vd.offset;
                nextView.leavers = validatedLeavers;
            }
            if ((IsisSystem.Debug & IsisSystem.VIEWCHANGE) != 0)
                Isis.WriteLine("UpdateGroupView<" + g.gname + (GroupIsReal ? "" : " TrackingProxy") + ">: nextView = " + nextView + ", InhibitActions=" + inhibitActions);
            if (inhibitActions)
            {
                nv = nextView;
                return;
            }
            if (GroupIsReal)
            {
                if ((g.flags & Group.G_ISLARGE) != 0 && g.theView != null)
                    nextView.NextIncomingMsgID[1] = g.theView.NextIncomingMsgID[1];
                long before = Isis.NOW();
                g.NewView(nextView, queue, null, ref nv);
                if ((IsisSystem.Debug & IsisSystem.DELAYS) != 0 && (Isis.NOW() - before) > Isis.ISIS_WARNAFTER)
                    Isis.WriteLine("WARNING: g.NewView delayed for " + (Isis.NOW() - before) + "ms");
                if (IAmLeaving == false)
                    g.ReplayToDo();
            }
            else // Tracking Proxy in the ORACLE
                using (new LockAndElevate(g.ViewLock))
                {
                    g.theView = nextView;
                    nv = nextView;
                    // Remember the value just in case we "later" need to use it to initialize a joining member
                    nv.nextMsgid = g.nextMsgid;
                }
            if (nextView.leavers.Length > 0)
                ReliableSender.PendingSendCleanup(g, nextView.leavers);
        }

        internal static Address[] Expand(Address[] list)
        {
            Group.vGroup vg = null;
            bool mustRevise = false;
            int newLen = list.Length;
            foreach (Address a in list)
                if (a.pid < 0 && (vg = Group.vGLookup(a)) != null)
                {
                    mustRevise = true;
                    newLen = newLen - 1 + vg.vGMembers.Length;
                }
            if (!mustRevise)
                return list;
            Address[] newList = new Address[newLen];
            int idx = 0;
            for (int i = 0; i < list.Length; i++)
                if (list[i].pid < 0 && ((vg != null & vg.vGroupAddr.Equals(list[i])) || (vg = Group.vGLookup(list[i])) != null))
                {
                    foreach (Address ma in vg.vGMembers)
                        newList[idx++] = ma;
                }
                else
                    newList[idx++] = list[i];
            return newList;
        }
    }
    /// <summary>
    /// A class containing various static definitions and methods used by the Isis^2 system.
    /// </summary>
    public static class IsisSystem
    {
        /// <summary>
        /// A flag application threads can monitor to sense Isis shutdown
        /// </summary>
        public static bool IsisActive = false;
        internal static bool IsisWasActive = false;
        internal static bool IsisAlreadyRan = false;
        internal static bool IsisRestarting = false;
        internal static bool fastStart = false;
        internal static int IsisJoinCounter = 0;

        internal const long GROUPEVENTS = 0x0000000000000001;
        internal const long MESSAGELAYER = 0x0000000000000002;
        internal const long REPLYWAIT = 0x0000000000000004;
        internal const long VIEWCHANGE = 0x0000000000000008;
        internal const long FRAGER = 0x0000000000000010;
        internal const long TYPESIGS = 0x0000000000000020;
        internal const long RELAYLOGIC = 0x0000000000000040;
        internal const long GVELOGIC = 0x0000000000000080;
        internal const long P2PLAYER = 0x0000000000000100;
        internal const long INTERFACES = 0x0000000000000200;
        internal const long LOCKSTATE = 0x0000000000000400;
        internal const long MCMD = 0x0000000000000800;
        internal const long STARTSEQ = 0x0000000000001000;
        internal const long VIEWWAIT = 0x0000000000002000;
        internal const long CALLBACKS = 0x000000000004000;
        internal const long FAILURES = 0x0000000000008000;
        internal const long DISCARDS = 0x0000000000010000;
        internal const long DALLOGIC = 0x0000000000020000;
        internal const long TOKENLOGIC = 0x0000000000040000;
        internal const long TOKENFLUSH = 0x0000000000080000;
        internal const long FLUSHING = 0x0000000000100000;
        internal const long VERBOSEADDRS = 0x0000000000200000;
        internal const long MSGIDS = 0x0000000000400000;
        internal const long AGGREGATION = 0x0000000000800000;
        internal const long LOCKCHECK = 0x0000000001000000;
        internal const long MCMDMAP = 0x0000000002000000;
        internal const long LOOPBACK = 0x0000000004000000;
        internal const long PPAYLOADS = 0x0000000008000000;
        internal const long DELIVERY = 0x0000000010000000;
        internal const long TCPOVERLAY = 0x0000000020000000;
        internal const long TUNNELING = 0x0000000040000000;
        internal const long TIMERS = 0x0000000080000000;
        internal const long FLOWCONTROL = 0x0000000100000000;
        internal const long CIPHER = 0x0000000200000000;
        internal const long DHTS = 0x0000000400000000;
        internal const long NACKS = 0x0000000800000000;
        internal const long DELAYS = 0x0000001000000000;
        internal const long LOWLEVELMSGS = 0x0000002000000000;      /* Caution: very verbose, uses in-memory storage.  Must call IsisSystem.WriteAckLog periodically or will leak memory */
        internal const long TOKENSTABILITY = 0x0000004000000000;
        internal const long CAUSALDELIVERY = 0x0000008000000000;
        internal const long DISKLOGGER = 0x0000010000000000;
        internal const long MSGQS = 0x0000020000000000;
        internal const long WARNIFSLOW = 0x0000040000000000;
        internal const long SPECIALDEBUG = 0x0000080000000000;
        internal const long LOCKS = 0x0000100000000000;
        internal const long SAFESEND = 0x0000200000000000;
        internal const long ORDEREDSEND = 0x0000400000000000;
        internal const long WARNABOUTSUBSETS = 0x0000800000000000;
        internal const long OOBXFERS = 0x0001000000000000;

        internal const long TOKENLAYER = FLUSHING | TOKENFLUSH | TOKENLOGIC;

        internal const long Debug = 0;

        /// <summary>
        /// Returns this process's address.  
        /// </summary>
        /// <returns></returns>
        public static Address GetMyAddress()
        {
            if (!IsisActive)
                throw new IsisException("Isis isn't running (did you forget to call Isis.Start()?)");
            return Isis.my_address;
        }

        /// <summary>
        /// An interface for obtaining a copy of the NULLADDRESS object
        /// </summary>
        /// <returns></returns>
        public static Address GetNullAddress()
        {
            return Isis.NULLADDRESS;
        }

        /// <summary>
        /// Returns a pretty-printable string giving the internal state of Isis on this node.  
        /// </summary>
        /// <returns></returns>
        public static string GetState()
        {
            GC.Collect();
            string state = "";
            if (IsisSystem.IsisActive == false)
                return ("ISIS is inactive");
            state = "\r\n----------------------- ISIS state for " + Isis.my_address + (Isis.ClientOf == null ? " [leaderId=" + Isis.LeaderId + "]" : "") + ":\r\n";
            for (int sc = 0; sc < 15; sc++)
            {
                try
                {
                    switch (sc)
                    {
                        case 0:
                            state += RunTimeStatsState();
                            break;
                        case 1:
                            state += Group.GetState();
                            break;
                        case 2:
                            state += Isis.GetTimerState();
                            break;
                        case 3:
                            state += Isis.RIPListState();
                            break;
                        case 4:
                            state += Isis.GetPLLState();
                            break;
                        case 5:
                            state += Isis.GetGVEState();
                            break;
                        case 6:
                            state += AwaitReplies.GetState();
                            break;
                        case 7:
                            state += FlowControl.GetState();
                            break;
                        case 8:
                            state += ReliableSender.GetState();
                            break;
                        case 9:
                            state += Group.deFragState();
                            break;
                        case 10:
                            state += ReliableSender.TCPListState();
                            break;
                        case 11:
                            state += Group.TunnelState();
                            break;
                        case 12:
                            state += MCMDSocket.GetState();
                            break;
                        case 13:
                            state += ILock.GetState();
                            break;
                        case 14:
                            state += BoundedBuffer.GetState();
                            break;
                    }
                }
                catch (Exception e)
                {
                    string[] what = { "RunTimeStates", "Group", "Timer", "RIPList", "PLL", "GVE", "Rdv", "FC", "ReliableSender", "deFrag", "TCPList", "Tunnels", "MCMDSocket", "Ilock", "BB" };
                    state += "Isis threw an internal exception < " + e + "> while trying to perform " + what[sc] + ".GetState()";
                }
            }
            state += "-------------------------End of State Dump------------------------------------\r\n";
            return state;
        }

        /// <summary>
        /// Returns a string summarizing the state of the Isis I/O subsystem
        /// </summary>
        /// <returns></returns>
        public static string GetIOState()
        {
            return RunTimeStatsState(); ;
        }

        internal static void WriteAckInfo()
        {

            if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                using (new LockAndElevate(ReliableSender.ackInfoLock))
                {
                    foreach (string aks in ReliableSender.ackInfo)
                        Isis.Write(aks);
                    ReliableSender.ackInfo = new List<string>();
                }
        }

        /// <summary>
        /// Runtime statistics for this version of Isis
        /// </summary>
        public class RuntimeStats
        {
            /// <summary>
            /// UDP packets sent
            /// </summary>
            public long UDPsent;
            /// <summary>
            /// UDP bytes sent
            /// </summary>
            public long UDPBsent;
            /// <summary>
            /// UDP packets received
            /// </summary>
            public long UDPrcvd;
            /// <summary>
            /// UDP bytes received
            /// </summary>
            public long UDPBrcvd;
            /// <summary>
            /// IPMC packets sent
            /// </summary>
            public long IPMCsent;
            /// <summary>
            /// IPMC bytes sent
            /// </summary>
            public long IPMCBsent;
            /// <summary>
            /// IPMC packets received
            /// </summary>
            public long IPMCrcvd;
            /// <summary>
            /// IPMC bytes received
            /// </summary>
            public long IPMCBrcvd;
            /// <summary>
            /// Tokens sent
            /// </summary>
            public long TokensSent;
            /// <summary>
            /// Tokens received
            /// </summary>
            public long TokensRcvd;
            /// <summary>
            /// TCP packets sent
            /// </summary>
            public long TCPsent;
            /// <summary>
            /// TCP bytes sent
            /// </summary>
            public long TCPBsent;
            /// <summary>
            /// TCP packets received
            /// </summary>
            public long TCPrcvd;
            /// <summary>
            /// TCP bytes received
            /// </summary>
            public long TCPBrcvd;
            /// <summary>
            /// Acks sent
            /// </summary>
            public long ACKsent;
            /// <summary>
            /// Acks received
            /// </summary>
            public long ACKrcvd;
            /// <summary>
            /// Nacks sent
            /// </summary>
            public long NACKsent;
            /// <summary>
            /// Nacks received
            /// </summary>
            public long NACKrcvd;
            /// <summary>
            /// Stability IPMC sent
            /// </summary>
            public long StabilitySent;
            /// <summary>
            /// Stability IPMC received
            /// </summary>
            public long StabilityRcvd;
            /// <summary>
            /// Token-triggered retransmissions
            /// </summary>
            public long TTRet;
            /// <summary>
            /// Total packets discarded as dups or for having no local receiver
            /// </summary>
            public long Discarded;
            internal LockObject Lock = new LockObject(false, "RTS.Lock");
            internal long rcvProcessingBeganAt;
            internal long ackedAt;
            internal long ackProcessingBeganAt;
#if WARN_ON_LONG_DELAYS
            internal long checkedAt;
#endif
            internal int[] ThreadCntrs = new int[34];

            internal void check()
            {
#if WARN_ON_LONG_DELAYS
                if (Isis.NOW() < 20000 || Isis.NOW() - checkedAt < 1000)
                    return;
                checkedAt = Isis.NOW();
                long rdelay = 0, a0delay = 0, a1delay = 0;
                using (new LockAndElevate(Lock))
                {
                    if (RTS.rcvProcessingBeganAt > 0)
                        rdelay = Isis.NOW() - RTS.rcvProcessingBeganAt;
                    if (RTS.ackedAt > 0)
                        a0delay = Isis.NOW() - RTS.ackedAt;
                    if (RTS.ackProcessingBeganAt > 0)
                        a1delay = Isis.NOW() - RTS.ackProcessingBeganAt;
                }

                if (rdelay > Isis.ISIS_WARNAFTER)
                    if (rdelay > Isis.ISIS_DEFAULTTIMEOUT*3)
                        throw new IsisException("EXCEPTION: receive thread has been processing a received message for " + rdelay);
                    else if ((IsisSystem.Debug & IsisSystem.WARNIFSLOW) != 0)
                        Isis.WriteLine("WARNING: receive thread has been processing a received message for " + rdelay + " doing " + Isis.ExtractStackTrace(ReliableSender.myP2PThread));
                if (a0delay > Isis.ISIS_WARNAFTER)
                    if (a0delay > Isis.ISIS_DEFAULTTIMEOUT*3)
                        throw new IsisException("EXCEPTION: ACK socket-handler thread has been trying to put a received ack on the ACKBB for " + a0delay + "ms");
                    else if ((IsisSystem.Debug & IsisSystem.WARNIFSLOW) != 0)
                        Isis.WriteLine("WARNING: ACK socket-handler thread has been trying to put a received ack on the ACKBB for " + a0delay + "ms");
                if (a1delay > Isis.ISIS_WARNAFTER)
                    if (a1delay > Isis.ISIS_DEFAULTTIMEOUT*3)
                        throw new IsisException("EXCEPTION: ACK processing thread has been processing a received ack/nack for " + a1delay + "ms");
                    else if ((IsisSystem.Debug & IsisSystem.WARNIFSLOW) != 0)
                        Isis.WriteLine("WARNING: ACK processing thread has been processing a received ack/nack for " + a1delay + "ms doing " + Isis.ExtractStackTrace(ReliableSender.myAckThread));
#endif
            }
        }

        /// <summary>
        /// System statistics
        /// </summary>
        public static RuntimeStats RTS = new RuntimeStats();

        internal static string RunTimeStatsState()
        {
            string s = "Summary of network statistics:\r\n    ";
            using (new LockAndElevate(IsisSystem.RTS.Lock))
            {
                s += "SENT: " + RTS.UDPsent + " UDP (" + RTS.UDPBsent + " non-duplicated bytes), " + RTS.TokensSent + " tokens, " + RTS.IPMCsent + " IPMC (" + RTS.IPMCBsent + " bytes; " + RTS.StabilitySent + " were stability packets), ";
                s += RTS.TCPsent + " TCP (" + RTS.TCPBsent + " bytes), " + RTS.ACKsent + " Acks, " + RTS.NACKsent + " Nacks.\r\n    ";
                s += "RECV: " + RTS.UDPrcvd + " UDP (" + RTS.Discarded + " were dups; " + RTS.UDPBrcvd + " bytes), " + RTS.TokensRcvd + " tokens, " + RTS.IPMCrcvd + " IPMC (" + RTS.IPMCBrcvd + " bytes; " + RTS.StabilityRcvd + " were stability packets), ";
                s += RTS.TCPrcvd + " TCP (" + RTS.TCPBrcvd + " bytes); " + RTS.Discarded + " were dups, " + RTS.ACKrcvd + " Acks, " + RTS.NACKrcvd + " Nacks, " + RTS.TTRet + " token-triggered resends\r\n";
            }
            return s;
        }

        /// <summary>
        /// Prints a line to the console and to the log
        /// </summary>
        public static void WriteLine(string s)
        {
            Isis.WriteLine(s);
        }

        /// <summary>
        /// Prints a word to the console and to the log
        /// </summary>
        public static void Write(string s)
        {
            Isis.Write(s);
        }

        internal static Thread ParentThread;

        /// <summary>
        /// Called to the Isis subsystem.  Must be your first call to Isis.
        /// </summary>
        public static void Start()
        {
            Start(false);
        }

        /// <summary>
        /// Called to Start the Isis subsystem.  Must be your first call to Isis.
        /// </summary>
        /// <param name="fast">If true, starts without searching for an already active Isis2 ORACLE</param>
        public static void Start(bool fast)
        {
            fastStart = fast;
            ParentThread = Thread.CurrentThread;
            if (IsisActive && !Isis.WORKER_MODE)
                throw new IsisException("Isis can't be started multiple times.");
            if (IsisAlreadyRan)
                throw new IsisException("Can't restart Isis v1.xx.xxxx after shutdown");
            Thread t = new Thread(delegate()
            {
                // .NET has a peculiar behavior when the class loader is still active: threads can be
                // created but they don't really run.  To work around that, Isis Start() itself runs in
                // a separate thread.  If this thread can start, other threads can start too!
                try
                {
                    Isis.ISIS_DEFAULTTIMEOUT <<= 1;
                    Thread.CurrentThread.Name = "IsisSystem.Start() initialization thread.";
                    IsisWasActive = IsisActive = true;
                    IsisAlreadyRan = true;
                    IsisRestarting = true;
                    Isis.timer_thread = new Thread(Isis.TimerThread);
                    Isis.timer_thread.Priority = ThreadPriority.Highest;
                    Isis.timer_thread.Name = "Isis timer callback thread";
                    Isis.timer_thread.Start();
                    ReliableSender.ResenderThreadLaunch();
                    if (Isis.my_IPaddress == null)
                        Isis.my_IPaddress = Isis.setMyAddress();
                    if (Isis.WORKER_MODE == false)
                    {
                        WaitUntilIsisIsRunning(true);
                        if (IsisRestarting || IsisActive == false)
                            throw new IsisException("Isis Start Failed(IsisRestarting=" + IsisRestarting + ", IsisActive=" + IsisActive + ")");
                    }
                    if (Isis.ISIS_LOGGED)
                    {
                        bool isis_mute = Isis.ISIS_MUTE;
                        Isis.ISIS_MUTE = true;
                        Isis.WriteLine("ISIS process-id: " + Isis.my_pid + ", my_address = " + Isis.my_address);
                        Isis.ISIS_MUTE = isis_mute;
                    }
                    Isis.SetupIM();
                    WaitUntilIsisIsRunning(false);
                    Isis.ISIS_DEFAULTTIMEOUT >>= 1;
                    if (Isis.WORKER_MODE)
                        Isis.ISISMEMBERS.Watch[Isis.MY_MASTER] += (Watcher)delegate(int ev) { throw new IsisException("master termination"); };
                }
                catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                IsisSystem.ThreadTerminationMagic();
            });
            t.Start();
            t.Join();
            if (Isis.ISISMEMBERS != null && Isis.ISISMEMBERS.HasFirstView && Isis.ISISMEMBERS.GetMyRank() > Isis.ISIS_MAXSYSTEMSIZE)
                Isis.WriteLine("WARNING: This version of Isis has only been tested with up to " + Isis.ISIS_MAXSYSTEMSIZE + " members.  Exceeding this threshold may trigger bugs or performance anomalies!");
        }

        /// <summary>
        /// Time since Isis start was called in milliseconds
        /// </summary>
        /// <returns>time</returns>
        public static long NOW()
        {
            return Isis.NOW();
        }

        /// <summary>
        /// Time since Isis start was called, in microseconds
        /// </summary>
        /// <returns>time</returns>
        public static long NOWus()
        {
            return Isis.NOWus();
        }

        private static int SafelyGetMyRank(Group g)
        {
            if (g == null || !g.HasFirstView)
                return -1;
            return g.theView.GetMyRank();
        }

        private static void WaitUntilIsisIsRunning(bool justOracle)
        {
            int howLong = Isis.WORKER_MODE ? 120 : 30;  // Seconds: Worker will wait 2 minutes; others 30 seconds
            howLong *= 4;
            for (int n = 0; n < howLong && IsisActive; n++)
            {
                if (IsisActive && (Isis.ClientOf != null || SafelyGetMyRank(Isis.ORACLE) != -1) && (justOracle || (SafelyGetMyRank(Isis.ISISMEMBERS) != -1 && ReliableSender.ResenderThread != null)))
                    return;
                Thread.Sleep(1000);
            }
            throw new IsisException("Isis Start Failed");
        }

        /// <summary>
        /// Declares to Isis that this process will run as a "worker" to a master process elsewhere in the datacenter.
        /// </summary>
        /// <param name="myMaster">Address of the "master" process that will call BatchStart(), in a verbose-format string (<see cref="Address.ToStringVerboseFormat()"/>)</param>
        /// <param name="timeoutms">Timeout in ms (ISISDEFAULTTIMEOUT if not specified)</param>
        /// <returns>The address the master should use to activate the worker, via a call to BatchStart().</returns>
        /// <remarks>When using RunAsWorker(), the workers will typically be launched on some large number of nodes, passing the Address of their
        /// master in as an argument to RunAsWorker().  The workers then call Isis.Start() and block until the master activates them by calling BatchStart().
        /// The master, in contrast, calls Isis.Start() and then RegisterAsMaster().  It launches the workers, giving them its Isis Address as an argument, and
        /// then waits until all the workers have started up; as each one calls RunAsWorker() the master receives an upcall providing the Address of that worker.
        /// When the master believes that all workers are running, it calls BatchStart() and this releases the workers, which will be waiting in Isis.Start().
        /// 
        /// If 60 seconds pass and the worker has not yet been "BatchStarted" by the master, it calls Shutdown.
        /// 
        /// Notice that there are some apparent race conditions here.  Isis handles them automatically: if calls to RunAsWorker() occur before the master
        /// calls RegisterAsMaster(), the incoming worker addresses are held and then upcalls occur once the RegisterAsWorker() method is finally called.
        /// 
        /// Isis provides no help on "how long to wait" for workers to start, but beware of problems in which you try to launch 1000 workers but somehow, only 992
        /// launch successfully.  The master won't receive 1000 callbacks no matter how long it waits and in such cases, your code just needs to handle the
        /// situation.  On most data centers, a 30 second wait should be long enough: if a worker hasn't registered by then, it probably won't.  The master
        /// can call RejectWorker() if it gets an upcall to its NewWorker callback after it no longer wants to receive additional such calls.
        /// </remarks>
        public static void RunAsWorker(string myMaster, int timeoutms)
        {
            Isis.WORKER_MODE = true;
            IsisSystem.IsisActive = true;
            Isis.MY_MASTER = new Address(myMaster);
            Isis.my_IPaddress = Isis.setMyAddress();
            Thread.Sleep(Isis.my_pid % 1000 * 30);    // Random delay of 0-30s to avoid overwhelming the master process with a huge burst of requests
            if (Isis.ISIS_TCP_ONLY)
                ReliableSender.TCPConnectAsWorker(Isis.MY_MASTER);
            ReliableSender.SendP2P(Msg.RUNASWORKER, Isis.MY_MASTER, null, new byte[0], true);
            Isis.OnTimer(timeoutms, delegate()
            {
                if (Isis.ISISMEMBERS == null || !Isis.ISISMEMBERS.HasFirstView || Isis.ISISMEMBERS.theView.GetMyRank() == -1)
                    Shutdown("RunAsWorker: taking too long");
            });
        }

        /// <summary>
        /// Overload for RunAsWorker(string myMaster, int timeoutms)
        /// </summary>
        /// <param name="myMaster">address of leader</param>
        public static void RunAsWorker(string myMaster)
        {
            RunAsWorker(myMaster, Isis.ISIS_DEFAULTTIMEOUT * 10);
        }

        internal static NewWorker MasterCallBack;

        /// <summary>
        /// The master uses this to register a callback that will be invoked once for each worker that calls RunAsWorker(), giving the Address of that worker
        /// Later when the master terminates, all its workers throw IsisException("Master has terminated").
        /// </summary>
        /// <param name="del"></param>
        public static void RegisterAsMaster(NewWorker del)
        {
            MasterCallBack = del;
        }

        /// <summary>
        /// Rejects a RunAsWorker() request.  The worker will throw an IsisException("received poison");
        /// </summary>
        /// <param name="who">The rejected worker's Address</param>
        public static void RejectWorker(Address who)
        {
            ReliableSender.SendPoison(who, Isis.my_address + " has rejected you as a worker");
        }

        /// <summary>
        /// Blocks until Isis terminates on this node, then throws an exception.  Used mostly in a dedicated ORACLE process:
        /// such a process calls Isis.Start() and then Isis.WaitForever().
        /// </summary>
        /// <remarks>When Isis terminates this will throw an exception, probably just IsisException("Shutdown") but perhaps something
        /// more colorful.  Until then, sits here, perhaps forever.</remarks>
        public static void WaitForever()
        {
            while (IsisSystem.IsisActive)
                Thread.Sleep(15000);
        }

        /// <summary>
        /// Used by a master process to activate a potentially long list of worker processes, which should call RunAsWorker(), pass their addresses to the master, and then block on calls to Isis.Start().
        /// The master process itself should first call Isis.Start() and only calls BatchStart() after collecting the addresses of the workers, usually via SendP2P().
        /// </summary>
        /// <remarks>When using RunAsWorker(), the workers will typically be launched on some large number of nodes, obtaining their addresses via RunAsWorker() and then
        /// passing these to the Master(), for example by appending them to a file that the master will read, passing them over a TCP link, or using the Isis SendP2P() API.
        /// The workers then call Isis.Start() and block until the master activates them by calling BatchStart().</remarks>
        public static void BatchStart(Address[] workers)
        {
            Address OracleLeader = Isis.ClientOf;
            OracleLeader = OracleLeader ?? Isis.ORACLE.theView.members[0];
            foreach (Address worker in workers)
                Isis.SendInitialOracleLeaderInfo(worker, OracleLeader);
            Group.multiJoin(workers, new Group[] { Isis.ISISMEMBERS });
            // Just in case, make sure all are listed in ISISMEMBERS before returning
            foreach (Address worker in workers)
            {
                int stry = 0;
                while (stry < 5 && Isis.ISISMEMBERS.GetRankOf(worker) == -1)
                    Thread.Sleep(1000);
                if (stry == 5)
                    throw new IsisException("BatchStart: Even after 5 seconds, worker " + worker + " wasn't listed in ISISMEMBERS");
            }
        }

        /// <summary>
        /// Master calls this to wait for workers to call WorkerSetupDone
        /// </summary>
        /// <param name="workers">worker processes</param>
        public static void WaitForWorkerSetup(List<Address> workers)
        {
            WaitForWorkerSetup(workers.ToArray());
        }

        /// <summary>
        /// Master calls this to wait for workers to call WorkerSetupDone
        /// </summary>
        /// <param name="workers">worker processes</param>
        public static void WaitForWorkerSetup(Address[] workers)
        {
            foreach (Address worker in workers)
            {
                try
                {
                    Isis.ISISMEMBERS.P2PQuery(worker, new Timeout(120000, Timeout.TO_ABORTREPLY, "INQUIRE"), Isis.INQUIRE);
                }
                catch (IsisAbortReplyException)
                {
                    throw new IsisException("Timeout on during WaitForWorkerSetup for worker " + worker);
                }
            }
        }

        internal static Semaphore waitForWorkerSetup = new Semaphore(0, 1);

        /// <summary>
        /// Worker calls this when any setup that needed to occur after Isis.Start is completed
        /// </summary>
        public static void WorkerSetupDone()
        {
            waitForWorkerSetup.Release();
        }

        /// <summary>
        /// Used by a master process to activate a potentially long list of worker processes, which should call RunAsWorker(), pass their addresses to the master, and then block on calls to Isis.Start().
        /// The master process itself should first call Isis.Start() and only calls BatchStart() after collecting the addresses of the workers, usually via SendP2P().
        /// </summary>
        /// <remarks>When using RunAsWorker(), the workers will typically be launched on some large number of nodes, obtaining their addresses via RunAsWorker() and then
        /// passing these to the Master(), for example by appending them to a file that the master will read, passing them over a TCP link, or using the Isis SendP2P() API.
        /// The workers then call Isis.Start() and block until the master activates them by calling BatchStart().</remarks>
        public static void BatchStart(List<Address> workers)
        {
            BatchStart(workers.ToArray());
        }

        internal static void CheckParentThread()
        {
            if (IsisActive && ParentThread.IsAlive == false)
                Shutdown("Isis Shutdown: Parent thread has exited");
        }

        /// <summary>
        /// Shuts Isis down.  In this release, restarting Isis is not supported.  Sorry!
        /// </summary>
        public static void Shutdown()
        {
            Shutdown(null);
        }

        internal volatile static bool shuttingDown = false;
        internal static object shutdownLock = new object();

        /// <summary>
        /// Shuts Isis down.  In this release, restarting Isis is not supported.  Sorry!
        /// Calls Environment.Exit unless you set ISIS_GRACEFULSHUTDOWN to true.
        /// </summary>
        /// <param name="s"></param>
        public static void Shutdown(string s)
        {
            lock (shutdownLock)
            {
                if (shuttingDown)
                    return;
                shuttingDown = true;
                if (!IsisSystem.IsisActive && IsisSystem.IsisWasActive)
                    return;
            }
            if (s != null)
                Isis.WriteLine(s);
            Dictionary<Address, Group> GClone = new Dictionary<Address, Group>();
            using (new LockAndElevate(Group.IsisGroupsLock))
                foreach (var kvp in Group.IsisGroups)
                    GClone.Add(kvp.Key, kvp.Value);
            if (Isis.ClientOf != null)
                FDInform(Isis.ClientOf);
            else if (Isis.ORACLE != null && Isis.ORACLE.HasFirstView)
            {
                foreach (Address a in Isis.ORACLE.theView.members)
                    if (!a.isMyAddress())
                    {
                        FDInform(a);
                        break;
                    }
            }
            IsisActive = false;
            if (!Isis.ISIS_GRACEFULSHUTDOWN)
            {
                Isis.WriteLine("Shutting down via Environment.Exit()");
                Environment.Exit(0);
            }
            else
            {
                /*  The code that follows tries to do a graceful shutdown of Isis, but has chronic "issues"... */
                try
                {
                    try
                    {
                        foreach (var kvp in GClone)
                        {
                            Group g = kvp.Value;
                            AwaitReplies.InterruptReplyWaits(g);
                            g.InterruptAggregationWaits();
                        }
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    try
                    {
                        BoundedBuffer.ShutDown();
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    try
                    {
                        ReliableSender.Shutdown();
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    LockAndElevate.Disable(true);
                    try
                    {
                        MCMDSocket.Shutdown();
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    try
                    {
                        ReliableSender.toResendSema.Release(1);
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    try
                    {
                        ILock.Shutdown();
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    try
                    {
                        Isis.my_logstream.Flush();
                    }
                    catch (IOException) { }
                }
                catch (Exception e)
                {
                    Console.WriteLine("While attempting to shut down Isis, threw " + e);
                }
            }
        }

        internal static void CheckLocksHeld()
        {
            GC.WaitForPendingFinalizers();
            if ((IsisSystem.Debug & IsisSystem.LOCKSTATE) != 0)
            {
                string s = LockObject.LocksIHold();
                if (s.Equals(" "))
                    return;
            }
        }

        internal static void ThreadTerminationMagic()
        {
            LockObject.ThreadIsTerminating();
        }

        // Attempt to trigger a very rapid failure notification
        internal static void FDInform(Address who)
        {
            if (Isis.ClientOf != null)
                Isis.ORACLE.doP2PSend(who, true, Isis.FDETECTION, Isis.my_address);
            else
                Isis.ORACLE.doSend(false, false, Isis.FDETECTION, Isis.my_address);
        }

        internal static void GotPoison(string why)
        {
            if (shuttingDown)
                return;
            IsisSystem.WriteAckInfo();
            Shutdown("Got poison (reason=\"" + why + "\") in " + IsisSystem.GetState());
            throw new IsisShutdown("poison");
        }
    }

    /// <summary>
    /// Appears in a Query argument list to separate the by-ref "results" parameters from the arguments to the
    /// Query method that will be invoked
    /// </summary>
    public class EOLMarker
    {
    }

    /// <summary>
    /// The Client class permits a non-member of a Group to relay requests through a "representative".
    /// </summary>
    public class Client
    {
        internal LockObject myRepLock;
        internal string gname;
        internal long[][] tsigs;
        internal Address myRep;
        internal static List<Client> Clients = new List<Client>();
        internal static LockObject ClientsLock = new LockObject("ClientsLock");

        /// <summary>
        /// Constructor for a Client stub object associated with the specified group name.  
        /// </summary>
        /// <remarks>
        /// A Client request will be satisfied even if the
        /// client process would not be allowed to join the actual group due to access control limits, so the group members must protect themselves
        /// against access by clients who are not authorized to access the group data.
        /// </remarks>
        /// <param name="gname">Group name</param>
        public Client(string gname)
        {
            if (!IsisSystem.IsisActive || IsisSystem.IsisRestarting)
                throw new IsisException(" New Client operation was requested but Isis wasn't running yet");
            using (new LockAndElevate(ClientsLock))
                foreach (Client cl in Clients)
                    if (cl.gname.Equals(gname))
                        throw new IsisClientException("Attempted to created two Client objects with the same group name");
            this.gname = gname;
            myRepLock = new LockObject(gname + ".ClientLock");
            if (!refreshRep())
                throw new IsisClientException("Client(" + gname + "): Group not found");
            // Disables checking of types on RPC vectored via ISISMEMBERS: a hack for the client of a group mode
            // This won't cause harm: ISISMEMBERS is only called from the Isis library and usually in type-checked mode
            // and the client's actual RPC call is type-checked in the Client class library
            Isis.ISISMEMBERS.isClientProxy = true;
            using (new LockAndElevate(ClientsLock))
                Clients.Add(this);
        }

        /// <summary>
        /// Check to see if there is an existing Client object for this group, return it if found
        /// </summary>
        /// <param name="gname">Group name</param>
        /// <returns>Client object if any, null if not known</returns>
        public static Client Lookup(string gname)
        {
            using (new LockAndElevate(ClientsLock))
                foreach (Client cl in Clients)
                    if (cl.gname.Equals(gname))
                        return cl;
            return null;
        }

        internal static void ResetRep(string gname, Address newRep)
        {
            using (new LockAndElevate(ClientsLock))
                foreach (Client cg in Clients)
                    if (cg.gname.Equals(gname))
                    {
                        using (new LockAndElevate(cg.myRepLock))
                            cg.myRep = newRep;
                        return;
                    }
        }

        private bool refreshRep()
        {
            byte[] ba;
            using (new LockAndElevate(myRepLock))
                myRep = null;
            if (Isis.ClientOf != null)
            {
                ba = Isis.ORACLE.doP2PQuery(Isis.ClientOf, new Timeout(Isis.ISIS_DEFAULTTIMEOUT, Timeout.TO_FAILURE, "BECLIENT"), Isis.BECLIENT, gname);
                if (ba.Length > 0)
                    using (new LockAndElevate(myRepLock))
                        myRep = (Address)Msg.BArrayToObjects(ba, typeof(Address))[0];
            }
            else using (new LockAndElevate(myRepLock))
                {
                    myRep = SelectHisRep(gname);
                    tsigs = GetTSigs(gname);
                }
            if (myRep != null)
            {
                ba = Isis.ISISMEMBERS.doP2PQuery(myRep, Isis.BECLIENT, gname);
                if (ba.Length > 0)
                    tsigs = (long[][])Msg.BArrayToObjects(ba, typeof(long[][]))[0];
                else using (new LockAndElevate(myRepLock))
                        myRep = null;
            }
            return myRep != null;
        }

        internal static int beClientCounter;

        internal static Address SelectHisRep(string gname)
        {
            Address hisRep = Isis.NULLADDRESS;
            Group g = Group.TrackingProxyLookup(gname);
            if (g != null && g.HasFirstView)
            {
                int rank = beClientCounter++ % g.theView.members.Length;
                while (rank < g.theView.hasFailed.Length && g.theView.hasFailed[rank])
                    ++rank;
                if (rank != g.theView.hasFailed.Length)
                    hisRep = g.theView.members[rank];
            }
            return hisRep;
        }

        internal static long[][] GetTSigs(string gname)
        {
            Group g = Group.Lookup(gname);
            if (g == null && Isis.ClientOf == null)
                g = Group.TrackingProxyLookup(gname);
            if (g == null)
                return new long[0][];
            long[][] tsvec = new long[g.Handlers.ListofhLists.Length][];
            for (int t = 0; t < tsvec.Length; t++)
                if (g.Handlers.ListofhLists[t] != null && g.allowsClientRequests[t])
                    tsvec[t] = ComputeTSig(g.Handlers.ListofhLists[t]);
                else
                    tsvec[t] = new long[0];
            return tsvec;
        }

        internal static long[] ComputeTSig(Group.myHandlers mh)
        {
            long[] rv = new long[mh.hList.Count()];
            int idx = 0;
            foreach (Group.CallBack cb in mh.hList)
            {
                ParameterInfo[] pi = cb.cbProc.hisCb.GetType().GetMethod("Invoke").GetParameters();
                string s = "";
                foreach (ParameterInfo pinfo in pi)
                    s += pinfo.ParameterType + ":";
                rv[idx++] = TSHash(s);
            }
            return rv;
        }

        /// <summary>
        /// Issues a peer-to-peer query to the group representative for this client, returns the result as a byte[] array.
        /// </summary>
        /// <remarks>
        /// P2PQueryToBA issues a peer-to-peer query to the group representative for this client, returns the result as a byte[] array.
        /// Normally, one would pass this array to Msg.BArrayToObjects or Msg.InvokeFromBArray.  The first argument must be
        /// the request identifier and the second argument an Isis.Timeout() object specifying how long to wait for a reply
        /// and what action to take if a timeout occurs.
        /// </remarks>
        /// <param name="given">Variable-length list specifying request, timeout, parameters.</param>
        /// <returns>byte[] vector encoding results, null if unsuccessful</returns>
        public byte[] P2PQueryToBA(params Object[] given)
        {
            Timeout timeout;
            Group.splitObs(null, out timeout, ref given);
            CheckTypeSig(given);
            byte[] ba = null;
            Address rep = null;
            int ntries = 0;
            while (findRep(ref rep, ref ntries) && (ba = Isis.ISISMEMBERS.doP2PQuery(rep, timeout, Isis.CLIENTWRAPPED, gname, (int)given[0], Msg.toBArray(given))).Length == 0)
                continue;
            return ba;
        }

        /// <summary>
        /// Issues a peer-to-peer query to the group representative for this client, returns 0 on failure, 1 if successful
        /// indicating how many replies were received. 
        /// </summary>
        /// <remarks>
        /// P2PQuery issues a peer-to-peer query to the group representative for this client, returns an integer value 
        /// indicating how many replies were received. The first argument must be
        /// the request identifier and the second argument an Isis.Timeout() object specifying how long to wait for a reply
        /// and what action to take if a timeout occurs. The replies themselves are stored into variables specified
        /// by reference in a list of by-ref reply objects that are separated from the argument list for the request handler
        /// by a special EOLMarker
        /// </remarks>
        /// <param name="given">Variable-length list specifying request, timeout, parameters, then an EOLMarker, then vectors in which to store results.</param>
        /// <returns>0 if unsuccessful, 1 on success</returns>
        public int P2PQuery(params object[] given)
        {
            Timeout timeout;
            object[] resRefs;
            Group.splitObs(null, out timeout, ref given, out resRefs);
            CheckTypeSig(given);
            byte[] ba = null;
            Address rep = null;
            int ntries = 0;
            while (findRep(ref rep, ref ntries) && (ba = Isis.ISISMEMBERS.doP2PQuery(rep, timeout, Isis.CLIENTWRAPPED, gname, (int)given[0], Msg.toBArray(given))).Length == 0)
                continue;
            if (ba.Length == 0)
                return 0;
            List<byte[]> bas = new List<byte[]>();
            bas.Add(ba);
            Msg.BArraysToLists(resRefs, bas);
            return 1;
        }

        /// <summary>
        /// Issues a multicast query to the group representative for this client, then invokes a delegate specified by the caller 
        /// in which received replies are proved as vectors of the corresponding object types, one entry per reply received. 
        /// </summary>
        /// <remarks>
        /// P2PQueryInvoke issues a multicast query to the group representative for this client, then invokes a delegate specified by the caller 
        /// in which received replies are proved as vectors of the corresponding object types, one entry per reply received. 
        /// The first argument must be
        /// the request identifier and the second argument an Isis.Timeout() object specifying how long to wait for a reply
        /// and what action to take if a timeout occurs. The delegate that will handle the replies is separated from the argument list for the request handler
        /// by a special EOLMarker</remarks>
        /// <param name="given">Variable-length list specifying request, timeout, parameters, then an EOLMarker, then vectors in which to store results.</param>
        public void P2PQueryInvoke(params Object[] given)
        {
            Timeout timeout;
            Group.splitObs(null, out timeout, ref given);
            CheckTypeSig(given);
            Delegate del = (Delegate)given[given.Length - 1];
            Isis.ArrayResize<object>(ref given, given.Length - 1);
            byte[] ba = null;
            Address rep = null;
            int ntries = 0;
            while (findRep(ref rep, ref ntries) && (ba = Isis.ISISMEMBERS.doP2PQuery(rep, timeout, Isis.CLIENTWRAPPED, gname, (int)given[0], Msg.toBArray(given))).Length == 0)
                continue;
            if (ba.Length == 0)
                return;
            Msg.InvokeFromBArray(ba, del);
        }

        // In this code we need to deal with the annoyance that NullReply causes us to get no replies, but so also would the failure of the
        // the selected representative.  So if findRep is called a second time or more times, it checks to see if the representative is
        // the same one.  If so, we return a NullReply to the user.  If not -- if we were assigned a new representative -- we reissue the
        // request.  But we do this no more than twice.
        internal bool findRep(ref Address rep, ref int ntries)
        {
            Address oldRep = rep;
            using (new LockAndElevate(myRepLock))
                rep = myRep;
            if (oldRep != null && rep != null && oldRep.Equals(rep))
                // NullReply case
                return false;
            if (rep == null)
            {
                if (!refreshRep())
                    // Group isn't running
                    return false;
                using (new LockAndElevate(myRepLock))
                    rep = myRep;
                if (oldRep != null && oldRep.Equals(rep))
                    // Not likely to occur, but call this another instance of the NullReply scenario
                    return false;
            }
            if (ntries++ == 2)
                throw new IsisAbortReplyException("This request is apparently causing group members to crash!");
            FlowControl.FCBarrierCheck();
            return true;
        }

        internal void CheckTypeSig(object[] obs)
        {
            int req = rcode(obs) + Isis.SYSTEMREQS;
            long mySig = ComputeTypeSig(obs);
            if (tsigs == null || myRep == null)
                throw new ArgumentException("Client intializer hasn't completed");
            if (req > tsigs.Length || tsigs[req].Length == 0)
                throw new ArgumentException("Group doesn't allow client calls to request code=" + rcode(obs));
            bool fnd = false;
            foreach (long ts in tsigs[req])
                if (ts == mySig)
                {
                    fnd = true;
                    break;
                }
            if (!fnd)
            {
                string ts = " ";
                for (int i = 1; i < obs.Length; i++)
                {
                    if (i > 1)
                        ts += ", ";
                    ts += obs[i].GetType().ToString();
                }
                throw new ArgumentException("Group doesn't have a handler for request code=" + req + " matching type signature (" + ts + " )");
            }
        }

        internal long ComputeTypeSig(object[] obs)
        {
            string sig = "";
            for (int i = 1; i < obs.Length; i++)
                sig += obs[i].GetType().ToString() + ":";
            return TSHash(sig);
        }

        private static long TSHash(string sig)
        {
            using (MemoryStream ms = new MemoryStream(Msg.StringToBytes(sig)))
            {
                using (HMAC hm = new HMACMD5(new byte[8] { 56, 78, 9, 23, 10, 87, 33, 11 }))
                {
                    byte[] ba = hm.ComputeHash(ms);
                    long rval = 0;
                    for (int i = 0; i < ba.Length; i++)
                        rval ^= (((long)ba[i]) & 0xFF) << ((i & 3) << 3);
                    return rval;
                }
            }
        }

        private static int rcode(object[] obs)
        {
            int rval;
            if (obs[0].GetType().Equals(typeof(byte)))
                rval = (byte)obs[0];
            else if (obs[0].GetType().Equals(typeof(int)))
                rval = (int)obs[0];
            else
                throw new ArgumentException("Client-to-group request: can't identify the request code");
            return rval;
        }

    }

    /// <exclude></exclude>
    public interface QKD
    {
        /// <exclude></exclude>
        Address[] GetDestsToArray(Group dht, int vid);
        /// <exclude></exclude>
        List<Address> GetDests(Group dht);
        /// <exclude></exclude>
        byte[] ToBArray();
    }

    /// <summary>
    /// Aggregation key for use when aggregating a DHT query
    /// </summary>
    [AutoMarshalled]
    public class QueryKey<KT> : IComparable<QueryKey<KT>>, IEquatable<QueryKey<KT>>, QKD
    {
        /// <exclude></exclude>
        public Address initiator;
        /// <exclude></exclude>
        public int uid;
        /// <exclude></exclude>
        public List<KT> keys;
        /// <exclude></exclude>
        public bool includeInitiator = true;

        private static int nextid;

        /// <summary>
        /// Constructor takes a list of keys
        /// </summary>
        /// <param name="keys">the keys used in this query</param>
        public QueryKey(IEnumerable keys)
            : this(Isis.my_address, keys, true)
        {
        }

        internal QueryKey(Address initiator, IEnumerable keys)
            : this(initiator, keys, true)
        {
        }

        internal QueryKey(Address initiator, IEnumerable keys, bool includeInitiator)
        {
            // We need to copy it into a form the Isis marshaller can represent...
            this.keys = new List<KT>();
            foreach (KT k in keys)
                this.keys.Add(k);
            if (this.keys.Count() == 0)
                throw new IsisException("QueryKey: key-list can't be empty");
            this.initiator = initiator;
            this.includeInitiator = includeInitiator;
            uid = ++nextid;
        }

        /// <exclude></exclude>
        public QueryKey()
        {
        }

        /// <exclude></exclude>
        public byte[] ToBArray()
        {
            return Msg.toBArray(keys, initiator, uid, includeInitiator);
        }

        /// <exclude></exclude>
        public QueryKey(List<KT> keys, Address initiator, int uid, bool includeInitiator)
        {
            this.keys = keys;
            this.initiator = initiator;
            this.uid = uid;
            this.includeInitiator = includeInitiator;
        }

        private int oldVid = -1;
        private List<Address> oldParticipants = null;
        private Address[] oldParticipantsAsArray = null;
        private LockObject opLock = new LockObject("opLock");

        /// <exclude></exclude>
        public Address[] GetDestsToArray(Group dht, int vid)
        {
            int ntries = 0;
            while (true)
            {
                using (new LockAndElevate(opLock))
                    if (vid == oldVid && oldParticipants != null)
                    {
                        if (oldParticipantsAsArray == null)
                            oldParticipantsAsArray = oldParticipants.ToArray();
                        return oldParticipantsAsArray;
                    }
                    else if (vid < oldVid)
                        return null;
                if (ntries++ > 0)
                    return null;
                GetDests(dht).ToArray();
            }
        }

        /// <exclude></exclude>
        public List<Address> GetDests(Group dht)
        {
            View theView;
            using (new LockAndElevate(dht.ViewLock))
                theView = dht.theView;
            using (new LockAndElevate(opLock))
                if (theView.viewid == oldVid && oldParticipants != null)
                    return oldParticipants;
            List<Address> dests = new List<Address>() { };
            if (includeInitiator)
                dests.Add(initiator);
            int iAg = dht.GetAffinityGroup(theView, initiator), kAg;
            int khash = initiator.GetHashCode();
            foreach (var key in keys)
                khash ^= key.GetHashCode();
            khash = Math.Abs(khash >> 3);
            foreach (var key in keys)
                if ((kAg = dht.GetAffinityGroup(dht.DHTKeyHash(key))) != iAg || !includeInitiator)
                {
                    Address dest;
                    int offset = khash % dht.myDHTBinSize * dht.myDHTnShards;
                    if (offset + kAg >= theView.members.Length)
                        offset = 0;
                    if (offset + kAg < theView.members.Length)
                    {
                        dest = theView.members[offset + kAg];
                        if (!dests.Contains(dest))
                            dests.Add(dest);
                    }
                    else if ((IsisSystem.Debug & IsisSystem.DHTS) != 0)
                        Isis.WriteLine("WARNING: GetDests encountered a depopulated affinity group for key=" + key);
                }
            using (new LockAndElevate(opLock))
            {
                oldVid = theView.viewid;
                oldParticipants = dests;
                oldParticipantsAsArray = null;
            }
            return dests;
        }

        ///<exclude></exclude>
        public override bool Equals(object obj)
        {
            if (obj == null || !obj.GetType().Equals(typeof(QueryKey<KT>)))
                return false;
            return Equals((QueryKey<KT>)obj);
        }

        ///<exclude></exclude>
        public bool Equals(QueryKey<KT> that)
        {
            return uid == that.uid && initiator.Equals(that.initiator);
        }

        ///<exclude></exclude>
        public int CompareTo(QueryKey<KT> that)
        {
            if (uid == that.uid)
                return initiator.CompareTo(that.initiator);
            return uid - that.uid;
        }

        /// <exclude></exclude>
        public override int GetHashCode()
        {
            int hc = initiator.GetHashCode() + uid * 77077;
            foreach (object key in keys)
                hc |= key.GetHashCode();
            return hc;
        }

        ///<exclude></exclude>
        public override string ToString()
        {
            string ks = " ";
            foreach (var k in keys)
                ks += "<" + k + "> ";
            return "QueryKey{" + ks + ", initator=" + initiator + (includeInitiator ? "(included)" : "(not included)") +
                (oldVid == -1 ? " (haven't yet computed participants)" : ".... participants for vid=" + oldVid + " are " +
                (oldParticipants == null ? "<<unknown>>" : (Address.VectorToString(oldParticipantsAsArray == null ? oldParticipants.ToArray() : oldParticipantsAsArray)) + "}"));
        }
    }


    /// <summary>
    /// The class corresponding to Isis process groups, which is a central feature of the Isis <it>virtual synchrony model</it>.
    /// </summary>
    public class Group : IComparable<Group>, IEquatable<Group>, IDisposable
    {
        internal static Dictionary<Address, Group> IsisGroups = new Dictionary<Address, Group>(100);                 // Groups this node either belongs to, or is joining (theView is null while joining!)
        internal static LockObject IsisGroupsLock = new LockObject("IsisGroupsLock", ThreadPriority.Highest);
        internal static Dictionary<Address, Group> TPGroups = new Dictionary<Address, Group>(100);                   // Tracking Proxy groups.  A kind of shadow used by the ORACLE to track membership info
        internal static LockObject TPGroupsLock = new LockObject("TPGroupsLock", ThreadPriority.Highest);
        internal static List<Address> GroupRIPList = new List<Address>();
        internal static LockObject GroupRIPLock = new LockObject("GroupRIPLock");

        // Tracked by all Isis nodes, for all groups
        internal LockObject groupLock;                                              // Use to lock/unlock entire group on entry from user 
        internal LockObject ViewLock = new LockObject("ViewLock", ThreadPriority.Highest);   // Protects g.theView
        internal LockObject TokenLock = new LockObject("TokenLock", ThreadPriority.Highest); // Protects g.theToken, in large groups
        volatile internal bool interesting = true;                                           // Send a token asap
        internal LockObject GroupFlagsLock = new LockObject("GroupFlagsLock");      // Touched often; hold very briefly!
        internal LockObject SIFLock = new LockObject("SIFLock", ThreadPriority.Highest);                    // Held while sending a message in fragments
        internal List<object[]> SIFList = null;                                     // Non-null while fragmenting a big message
        internal Semaphore CPSSema;                                                 // If waiting on a flush
        volatile internal View theView;                                             // Current 
        internal View nextView;                                                     // Non-null during the state transfer logic
        internal Initializer theInitializer = null;
        volatile internal bool initializationDone = false;
        internal String gname;
        internal String where;
        internal Address gaddr;

        // Fields active only when groupClosed is false
        internal Thread groupReaderThread;
        internal Thread groupP2PReaderThread;
        internal Thread groupIPMCReaderThread;
        volatile internal bool GroupOpen = false;                                   // Becomes false once group.leave()/group.Terminate() is called
        internal bool TermSent = false;
        internal bool WasOpen = false;                                              // True if the group has ever been open
        internal bool isTrackingProxy = false;
        internal bool isClientProxy = false;
        internal bool HasFirstView = false;                                         // Becomes true when I get my very first view
        internal bool CallbacksDone = false;                                        // Becomes true after the NEWVIEW callbacks (if any) for the first view
        internal int myVirtIPAddr = MCMDSocket.UNKNOWN;                             // IP address for this group as a long int
        internal bool hasPhysMapping = false;                                       // Used only for tracking proxies: group has an assigned IPMC address
        internal int myPhysIPAddr = MCMDSocket.UNKNOWN;                             // Again, used in tracking proxy to represent the assigned IPMC address
        internal MCMDSocket my_socket;
        volatile internal int nextMsgid = 0;                                        // Sequential for group multicasts on a per-group basis
        volatile internal int nRaw = 0;                                             // Counts raw messages sent in a row
        internal int lastLgAckID = -1;
        internal int rcvdMcastsCnt = 0;                                             // Incremented on received multicasts, zeroed every five minutes
        internal int rcvdMcastsRate = 0;                                            // Running average: newAvg = (newRate + oldAvg)/2;
        internal bool getsMCADDR;                                                   // True in a tracking proxy at the MCMD deamon if we're allocating a true IPMC address to this group
        internal Group sameAs;                                                      // If non-null in a tracking proxy at the MCMD deamon, tells us to use the same IPMC address as the sameAs group
        internal bool LeaderMode = false;                                           // Becomes true once this node is the active leader for the ORACLE
        internal bool TakingOver = false;                                           // True when the old leader fails and I'm in the process of taking over from him
        internal int myFirstLeadershipView;                                         // ViewID when I first took over as large group owner (else 0)
        internal int safeSendThreshold = ALL;                                       // The number of nodes that must ack a SafeSend before it will be delivered
        internal durabilityMethod safeSendDurabilityMethod;                         // The method to use for making a SafeSend durable.  If none is specified, in-memory caching is used
        internal List<KeyValuePair<Thread, Msg>> curMsgList = new List<KeyValuePair<Thread, Msg>>();
        internal LockObject curMsgListLock = new LockObject("curMsgListLock", ThreadPriority.Highest);
        internal List<Msg> Unstable = new List<Msg>();                              // Lists messages that aren't yet stable because not yet fully acknowledged
        internal LockObject UnstableLock = new LockObject("UnstableLock");                                // Protects Unstable list
        internal long SentStableAt;                                                 // Isis.NOW when the last multicast to this group occured
        internal bool SendStabilityNeeded;                                          // True if some form of stability message needs to be sent
        internal int MaxBacklogSent;                                                // Largest value of LocalBacklog sent since last STABILITYINFO multicast
        internal BoundedBuffer incomingSends;
        internal BoundedBuffer incomingP2P;

        internal Semaphore Wedged = new Semaphore(1, int.MaxValue);                 // Used to wedge group during membership changes
        internal ILFunc myLoggingFcn;                                               // If logged, routine to call back to on events of interest
        internal long myLoggingId;

        internal const int IL_SEND = 0;
        internal const int IL_SAFESEND = 1;
        internal const int IL_ORDEREDSEND = 2;
        internal const int IL_QUERY = 3;
        internal const int IL_DELIVERY = 4;
        internal const int IL_AGGWAIT = 5;
        internal const int IL_AGGDVALUE = 6;
        internal const int IL_AGGLVALUE = 7;
        internal const int IL_AGGREGATE = 8;

        internal const int IL_START = 0;
        internal const int IL_DONE = 1;

        volatile internal int flags;                                                // Flags associated with this group

        internal LockObject quiesceLock = new LockObject("quiesceLock");
        internal int quiesceCnt;
        internal Semaphore quiesceWait = new Semaphore(0, int.MaxValue);

        // These flags come in two flavors.  Some are "set once", when the group is first created and for those we don't need to worry about locking before testing.
        // But the others are turned on and off and on them, you must obtain the right kind of lock before modifying or reading.  
        internal const int G_ISLARGE = 0x01;                                        // [set once, no lock needed] Group is large
        internal const int G_USEUNICAST = 0x02;                                     // [set once, no lock needed] Unicast only
        internal const int G_USEIPMC = 0x04;                                        // [set once, no lock needed] Use IPMC if possible
        internal const int G_HASUAGG = 0x08;                                        // [set once, no lock needed] Contains one or more user defined aggregators, run tokens faster
        internal const int G_TRACE = 0x10;                                          // [set once, no lock needed] Trace events in this group
        internal const int G_SECURE = 0x20;                                         // [set once, no lock needed] True for a secure group
        internal const int G_ISRAW = 0x40;                                          // [set once, no lock needed] Disables vsync flush
        internal const int G_TERMINATING = 0x80;                                    // [needs quiesceLock] Preparing to close/terminate this group
        internal const int G_WEDGED = 0x100;                                        // [needs GroupFlagsLock] True while group is wedged for membership changes
        internal const int G_SENDINGSTABILITY = 0x0200;                             // [needs GroupFlagsLock] True if the group is in the process of sending its stability report
        internal const int G_NEEDSTATEXFER = 0x400;                                 // [needs GroupFlagsLock] Wait until state xfer completes before delivering first message
        internal const int G_USESOOB = 0x800;                                       // Uses OOB subsystem

        internal static string[] flagNames = new string[] { "islarge", "useunicast", "useIPMC", "hasuagg", "trace", "secure", "raw", "terminating", "wedged", "sendingstability", "needstatexfer", "usesOOB" };

        internal Isis.PendingLeaderOps NotifyDALOnReply = null;                     // Non-null if the "do as leader" code is running
        internal int uids;                                                          // Unique ids used in connection with the DAL logic
        internal List<Msg> ToDo = new List<Msg>();                                  // Received and already acked, but had a future view-id
        internal LockObject ToDoLock = new LockObject("ToDoLock");
        internal List<byte[]> IPMCArrivedEarly = new List<byte[]>();                // Received and not acked; arrived during the first stages of group join
        internal List<ReliableSender.MReplayMe> MsgArrivedEarly = new List<ReliableSender.MReplayMe>();
        internal List<Msg> OutOfOrderQueue = new List<Msg>();
        internal LockObject OutOfOrderQueueLock = new LockObject("outOfOrderQueueLock");
        internal List<IPMCVinfo> stashedIPMCviews = new List<IPMCVinfo>();
        internal LinkedList<object>[] AggList;                                      // For small-group aggregation: for each level, a list of the associated aggregations
        internal LockObject AggListLock = new LockObject("AggListLock");
        internal class svi
        {
            internal Address sender;
            internal int vid;
            internal int msgid;

            internal svi(Address s, int v, int m)
            {
                sender = s; vid = v; msgid = m;
            }
        }
        internal List<svi> desiredOrderQueue = new List<svi>();
        volatile internal tokenInfo theToken;                                       // My representation of this group's token
        internal int gcollectedTo = -1;                                             // Last value of aggstable used in garbage collecting lgPendingSendBuffer
        internal long TypeSig;
        internal string TypeSigStr = "<undef>";
        internal bool joinFailed = false;                                           // Relays info about join failure
        internal string reason = "";                                                // If join fails, this will give the reason

        internal string myCheckpointFile;                                           // If non-null, the file name into which checkpoints should be saved
        internal int CheckpointFrequency = -1;                                      // If greater than 0, an interval, in ms, at which checkpoints should be made
        internal FileStream myChkptStream = null;                                   // While checkpointing, the temporary file into which the checkpoint is being written
        internal byte[] myAESkey = null;                                            // A secret key for this group, if the group uses AES security
        internal bool userSpecifiedKey = false;                                     // True if the end-user specified the key
        internal Aes myAes = null;
        internal LockObject myAesLock = new LockObject("myAESLock");
        internal ICryptoTransform myDecryptor = null;

        /// <exclude>
        /// <summary>
        /// Internal Aggregation structure
        /// </summary>
        /// </exclude>
        public class AggInfo : IComparable
        {
            internal Type KT;
            internal Type VT;
            internal string KVT;
            // This is tricky.  The following is actually a reference to an object of type 
            // Aggregator<KeyType,ValueType> that implements the IAggregateEventHandler interface
            internal ConstructorInfo myFactory;
            // to pass into the factory
            internal object theGroup;
            internal object theDel;
            // And the "last resort" action to take...
            internal Timeout theTimeout;

            internal AggInfo(Type kt, Type vt, ConstructorInfo mc, object g, object d, Timeout theTO)
            {
                KT = kt;
                VT = vt;
                KVT = KT.ToString() + ":" + VT.ToString();
                myFactory = mc;
                theGroup = g;
                theDel = d;
                theTimeout = theTO;
            }

            /// <exclude>
            /// <summary>
            /// Required public constructor
            /// </summary>
            /// </exclude>
            public AggInfo()
            {
            }

            /// <exclude>
            /// <summary>
            /// Required equality test
            /// </summary>
            /// </exclude>
            public override bool Equals(object obj)
            {
                if (obj == null || !obj.GetType().Equals(typeof(AggInfo)))
                    return false;
                return VT.Equals((AggInfo)obj) && KT.Equals((AggInfo)obj);
            }

            /// <exclude>
            /// <summary>
            /// Required hash code computation
            /// </summary>
            /// </exclude>
            public override int GetHashCode()
            {
                return VT.GetHashCode() * 37 + KT.GetHashCode();
            }

            /// <exclude>
            /// <summary>
            /// Required comparator
            /// </summary>
            /// </exclude>
            public int CompareTo(object obj)
            {
                return KVT.CompareTo(((AggInfo)obj).KVT);
            }
        }

        internal List<AggInfo> AggTypes = new List<AggInfo>();                      // Aggregator types registered in this group

        internal Address[] Dying = new Address[0];                                  // Leaving the Group
        internal Address[] Joining = new Address[0];                                // Joining the Group

        /// <summary>
        /// ALL members must reply
        /// </summary>
        public const int ALL = -1;
        /// <summary>
        /// At least a majority must reply
        /// </summary>
        public const int MAJORITY = -2;

        /// <summary>
        /// End of the list of arguments, start of the list of reply variables
        /// </summary>
        public static EOLMarker EOL = new EOLMarker();          // Used as a marker between arguments and reply variables list in Query

        /// <exclude>
        /// <summary>
        /// Internal to Isis; public to comply with C# scope rules
        /// </summary>
        /// <param name="b"></param>
        /// <returns></returns>
        /// </exclude>
        public int CompareTo(Group b)
        {
            if (b == null)
                return -1;
            return gaddr.CompareTo(b.gaddr);
        }

        /// <exclude>
        /// <summary>
        /// Internal to Isis; public to comply with C# scope rules
        /// </summary>
        /// <param name="b"></param>
        /// <returns></returns>
        /// </exclude>
        public bool Equals(Group b)
        {
            return gaddr.Equals(b.gaddr);
        }

        /// <exclude>
        /// <summary>
        /// Callback class, internal
        /// </summary>
        /// </exclude>
        public class CallBack
        {
            internal bool withLock;
            internal Callable cbProc;

            /// <exclude>
            /// <summary>
            /// Callback constructor, internal
            /// </summary>
            /// </exclude>
            public CallBack(bool wl, Delegate d) { withLock = wl; cbProc = new Callable(d); }
        }

        internal class VHCallBack
        {
            internal bool withLock;
            internal ViewHandler vhProc;

            // Has ability to obtain the LLENTRY lock on gaddr before calling back but currently Isis isn't
            // using that feature because it was provoking deadlocks.  But must worry now about vsync violations
            internal VHCallBack(bool wl, ViewHandler vh) { withLock = wl; vhProc = vh; }
        }

        internal class UHCallBack
        {
            internal bool withLock;
            internal UCallback uhProc;

            // Has ability to obtain the LLENTRY lock on gaddr before calling back but currently Isis isn't
            // using that feature because it was provoking deadlocks.  But must worry now about vsync violations
            internal UHCallBack(bool wl, UCallback vh) { withLock = wl; uhProc = vh; }
        }

        /// <exclude>
        /// <summary>
        /// Internal to Isis; public to comply with C# scope rules
        /// </summary>
        /// </exclude>
        public class myHandlers
        {
            internal List<CallBack> hList = new List<CallBack>();
            /// <exclude>
            /// <summary>
            /// Internal to Isis; public to comply with C# scope rules
            /// </summary>
            /// </exclude>
            public static myHandlers operator +(myHandlers a, Delegate b)
            {
                if (a == null)
                    a = new myHandlers();
                a.hList.Add(new CallBack(false, b));
                return a;
            }

            /// <exclude>
            /// <summary>
            /// Internal to Isis; public to comply with C# scope rules.  This is for the doRegister case
            /// </summary>
            /// </exclude>
            public static myHandlers operator +(myHandlers a, CallBack b)
            {
                if (a == null)
                    a = new myHandlers();
                a.hList.Add(b);
                return a;
            }
        }

        /// <exclude>
        /// <summary>
        /// Internal to Isis; public to comply with C# scope rules
        /// </summary>
        /// </exclude>
        public class myWatches
        {
            internal List<Watcher> hList = new List<Watcher>();
            /// <exclude>
            /// <summary>
            /// Internal to Isis; public to comply with C# scope rules
            /// </summary>
            /// </exclude>
            public static myWatches operator +(myWatches a, Watcher b)
            {
                if (a == null)
                    a = new myWatches();
                a.hList.Add(b);
                return a;
            }
            /// <exclude>
            /// <summary>
            /// Internal to Isis; public to comply with C# scope rules
            /// </summary>
            /// </exclude>
            public static myWatches operator -(myWatches a, Watcher b)
            {
                if (a == null)
                    a = new myWatches();
                a.hList.Remove(b);
                return a;
            }
        }

        /// <exclude>
        /// <summary>
        /// Internal to Isis; public to comply with C# scope rules
        /// </summary>
        /// </exclude>
        public class myVHandlers
        {
            internal List<VHCallBack> vhList = new List<VHCallBack>();
            internal LockObject vhListLock = new LockObject("vhListLock");
            /// <exclude>
            /// <summary>
            /// Internal to Isis; public to comply with C# scope rules
            /// </summary>
            /// </exclude>
            public static myVHandlers operator +(myVHandlers a, ViewHandler b)
            {
                using (new LockAndElevate(a.vhListLock))
                    a.vhList.Add(new VHCallBack(false, b));
                return a;
            }

            // Puts handler at the front of the queue.  For internal use only!
            /// <exclude>
            /// <summary>
            /// Internal to Isis; public to comply with C# scope rules
            /// </summary>
            /// </exclude>
            public static myVHandlers operator *(myVHandlers a, ViewHandler b)
            {
                List<VHCallBack> newList = new List<VHCallBack>();
                newList.Add(new VHCallBack(false, b));
                foreach (VHCallBack vhcb in a.vhList)
                    newList.Add(vhcb);
                using (new LockAndElevate(a.vhListLock))
                    a.vhList = newList;
                return a;
            }
        }

        /// <exclude>
        /// <summary>
        /// Internal to Isis; public to comply with C# scope rules
        /// </summary>
        /// </exclude>
        public class myUMHandlers
        {
            internal bool locked;
            internal List<UHCallBack> uhList = new List<UHCallBack>();
            internal LockObject uhListLock = new LockObject("uhListLock");
            /// <exclude>
            /// <summary>
            /// Internal to Isis; public to comply with C# scope rules
            /// </summary>
            /// </exclude>
            public static myUMHandlers operator +(myUMHandlers a, UCallback b)
            {
                if (a.locked)
                    throw new IsisException("Illegal attempt to register a handler after calling Group.Join()");
                a.uhList.Add(new UHCallBack(false, b));
                return a;
            }
        }

        /// <exclude>
        /// <summary>
        /// Internal to Isis; public to comply with C# scope rules
        /// </summary>
        /// </exclude>
        public class myUP2PHandlers
        {
            internal bool locked;
            internal List<UHCallBack> uhList = new List<UHCallBack>();
            internal LockObject uhListLock = new LockObject("uhListLock");
            /// <exclude>
            /// <summary>
            /// Internal to Isis; public to comply with C# scope rules
            /// </summary>
            /// </exclude>
            public static myUP2PHandlers operator +(myUP2PHandlers a, UCallback b)
            {
                if (a.locked)
                    throw new IsisException("Illegal attempt to register a handler after calling Group.Join()");
                a.uhList.Add(new UHCallBack(false, b));
                return a;
            }
        }

        /// <exclude>
        /// <summary>
        /// Internal to Isis; public to comply with C# scope rules
        /// </summary>
        /// </exclude>
        public class mimicVector
        {
            internal bool locked;
            /// <exclude>
            /// <summary>
            /// Internal to Isis; public to comply with C# scope rules
            /// </summary>
            /// </exclude>
            public myHandlers[] ListofhLists = new myHandlers[0];
            /// <exclude>
            /// <summary>
            /// Internal to Isis; public to comply with C# scope rules
            /// </summary>
            /// <param name="i"></param>
            /// <returns></returns>
            /// </exclude>
            public myHandlers this[int i]
            {
                get
                {
                    i += Isis.SYSTEMREQS;
                    if (i >= ListofhLists.Length)
                        Isis.ArrayResize<myHandlers>(ref ListofhLists, i + 1);
                    return ListofhLists[i];
                }
                set
                {
                    if (locked)
                        throw new IsisException("Illegal attempt to register a handler after calling Group.Join()");
                    i += Isis.SYSTEMREQS;
                    ListofhLists[i] = value;
                }
            }
        }

        /// <exclude>
        /// <summary>
        /// Internal to Isis; public to comply with C# scope rules
        /// </summary>
        /// </exclude>
        public class mimicWVector
        {
            /// <exclude>
            /// <summary>
            /// Internal to Isis; public to comply with C# scope rules
            /// </summary>
            /// </exclude>
            public Dictionary<Address, myWatches> ListofwLists = new Dictionary<Address, myWatches>(8);
            private LockObject ListofwListsLock = new LockObject("ListofwListsLock");
            /// <exclude>
            /// <summary>
            /// Internal to Isis; public to comply with C# scope rules
            /// </summary>
            /// <param name="who"></param>
            /// <returns></returns>
            /// </exclude>
            public myWatches this[Address who]
            {
                get
                {
                    using (new LockAndElevate(ListofwListsLock))
                    {
                        myWatches mw;
                        if (ListofwLists.TryGetValue(who, out mw))
                            return mw;
                        else
                        {
                            ListofwLists.Add(who, new myWatches());
                            return ListofwLists[who];
                        }
                    }
                }
                set
                {
                    using (new LockAndElevate(ListofwListsLock))
                    {
                        if (ListofwLists.ContainsKey(who))
                            ListofwLists[who] = value;
                        else
                            ListofwLists.Add(who, value);
                    }
                }
            }
        }

        /// <summary>
        /// A vector of handlers for incoming requests and updates, to which users attach additional handlers for requests defined by their code.
        /// </summary>
        /// <remarks>
        /// To register a request handler, one can call the explicit RegisterHandler method, but it is also possible (and
        /// more elegant) to write code like this:  Handlers[request-id] += (myHandler)delegate(... args ...) { ..... code to handle requests with matching type signature .... };
        /// Your code will be invoked later, each time a new message to the group is received and matches the request-id and type signature of your handler.
        /// </remarks>
        public mimicVector Handlers = new mimicVector();

        /// <summary>
        /// A vector of handlers for watched processes.
        /// </summary>
        /// <remarks>
        /// To register a watch on a group member, write code like this:  <c>Watch[who] += (Isis.Watcher)delegate(int event) { };</c>  
        /// If you may need to cancel the watch, create a method and register it as follows: <c>Watch[who] += myWatcherMethod; </c>
        /// In this case, you can use  <c>Watch[who] -= myWatcherMethod; </c> to cancel the watch.
        /// The event indicates whether the watched process joined the group (W_JOIN) or left the group (W_LEAVE).
        /// </remarks>
        /// 
        public mimicWVector Watch = new mimicWVector();

        internal SortedList<long, Msg>[] PendingQueue;
        internal LockObject PendingQueueLock = new LockObject("PendingQueueLock", ThreadPriority.Highest);

        internal ChkptChoser theChkptChoser;
        internal List<ChkptMaker> theChkptMakers = new List<ChkptMaker>();
        internal volatile bool inhibitEOC = false;

        /// <summary>
        /// New view callback handlers.
        /// </summary>
        /// <remarks>
        /// To register a ViewHandler, one can call the explicit RegisterViewCB method, but it is also possible (and
        /// more elegant) to write code like this:  ViewHandlers += (VHandler)delegate(View v) { ..... code to handle new views .... };
        /// </remarks>
        public myVHandlers ViewHandlers = new myVHandlers();
        // The FlushHandler callback is for internal use only; an upcall occurs during PROPOSE but the code needs to be live
        // or group view updates will wedge.  Thus I don't consider this to be a safe call to expose to Isis users
        internal myVHandlers FlushHandlers = new myVHandlers();
        internal LockObject ViewHandlersLock = new LockObject("ViewHandlersLock");

        internal bool hasUniversalHandlers, isP2PThread;
        internal myUMHandlers UniversalMHandlers = new myUMHandlers();
        internal myUP2PHandlers UniversalP2PHandlers = new myUP2PHandlers();

        internal Group(string gname, Address gaddr, View v)
        {
            if (!IsisSystem.IsisActive)
                throw new IsisException("Must call IsisSystem.Start() first");
            MakeChkpt = new ChkptMkr(this);
            LoadChkpt = new ChkptLdr(this);
            Initializer = new Initer(this);
            this.gname = gname ?? "<unknown>";
            this.gaddr = gaddr;
            using (new LockAndElevate(Group.GroupRIPLock))
                if (Group.GroupRIPList.Contains(gaddr))
                    throw new IsisException("Must not rejoin/recreate a group immediately after leave/terminate");
            AddToGroupsList(gname, gaddr);
            Setup();
            NewView(v, "IsisGroup create", null);
        }

        private void AddToGroupsList(string gname, Address gaddr)
        {
            using (new LockAndElevate(IsisGroupsLock))
            {
                foreach (var kvp in IsisGroups)
                {
                    Group g = kvp.Value;
                    if (g.gname.Equals(gname) || g.gaddr.Equals(gaddr) || g == this)
                        throw new IsisException("AddToGroupsList(" + gname + "/" + gaddr + "): but this group name or gaddr already exists (" + g.gname + "/" + g.gaddr + ")");
                }
                // These need to be big enough to deal with arriving fragmented messages, which can create big bursty loads
                incomingSends = new BoundedBuffer(gname + ":DeliverInOrder(IPMC)", 512, ILock.LLDELIVERY, -1, -1);
                incomingP2P = new BoundedBuffer(gname + ":DeliverInOrder(P2P)", 512, ILock.LLDELIVERY, -1, -1);
                if (IsisGroups.ContainsKey(gaddr))
                    throw new IsisException("AddToGroupsList: already listed");
                IsisGroups.Add(gaddr, this);
            }
        }

        /// <summary>
        /// The most standard way of constructing a new Group object: the user specifies the name and let's Isis fill in other data.
        /// Once the Group handle is obtained, it is important to register handlers, data types, aggregators, etc, prior to calling Join or Create.
        /// </summary>
        /// <param name="name">Group name</param>
        public Group(String name)
        {
            if (!name.Equals("ORACLE") && !name.Equals("ISISMEMBERS") && (!IsisSystem.IsisActive || IsisSystem.IsisRestarting))
                throw new IsisException("Group Join/Create operation was requested but Isis wasn't running yet");
            MakeChkpt = new ChkptMkr(this);
            LoadChkpt = new ChkptLdr(this);
            Initializer = new Initer(this);
            groupLock = new LockObject("<" + name + ">.GroupLock", ThreadPriority.Highest);
            Bind(name);

            // Group is uniquely named by hashing its name to a virtual IPMC address, but won't use IPMC unless
            // we have permission, and even then, only if Dr. Multicast (the MCMD) assigns a physical IP address 
            // to this group (or to some set of groups that includes this one)
            long addr = (Isis.CLASSD + Isis.ISIS_MCRANGE_LOW + Address.GroupNameHash(gname)) & 0xFFFFFFFFL;
            gaddr = new Address(Isis.LastIPv4(MCMDSocket.PMCAddr((int)addr)), 0);
            using (new LockAndElevate(Group.GroupRIPLock))
                if (Group.GroupRIPList.Contains(gaddr))
                    throw new IsisException("Must not rejoin/recreate a group immediately after leave/terminate");
            AddToGroupsList(gname, gaddr);
            Setup();
        }

        private Group()
        {
            MakeChkpt = new ChkptMkr(this);
            LoadChkpt = new ChkptLdr(this);
            Initializer = new Initer(this);
            groupLock = new LockObject("<anonymous group>.Lock");
        }

        volatile internal bool[] allowsClientRequests = new bool[512];

        /// <summary>
        /// Enables a handler to accept P2PQuery requests from group clients
        /// </summary>
        /// <param name="request">The request code</param>
        public void AllowClientRequests(int request)
        {
            request += Isis.SYSTEMREQS;
            if (request >= 0 && request <= 511)
                allowsClientRequests[request] = true;
            else
                throw new IsisException("illegal request code");
        }

        /// <summary>
        /// Redirects the role of representating this client to the specified new representative.  Pending requests will be reissued.
        /// </summary>
        /// <param name="Client">A client of the group currently bound to the caller</param>
        /// <param name="newRepresentative">A group member that will handle this client's requests in the future</param>
        public void RedirectClient(Address Client, Address newRepresentative)
        {
            if (GetRankOf(newRepresentative) == -1 || GetRankOf(Client) != -1 || Isis.ISISMEMBERS.GetRankOf(Client) == -1)
                throw new ArgumentException("Illegal argument to Isis.RedirectClient");
            Isis.ISISMEMBERS.doP2PSend(Client, true, Isis.BECLIENT, gname, newRepresentative);
        }

        /// <exclude>
        /// <summary>
        /// Internal, pretty-prints information about tracking proxies in ORACLE members
        /// </summary>
        /// <param name="tpg">Tracking proxy</param>
        /// <returns>Group state as a string</returns>
        /// </exclude>
        public static string TPtoString(Group tpg)
        {
            string s = ((tpg.flags & G_ISLARGE) != 0 ? "Large: " : "") + "<" + tpg.gname + "/" + tpg.gaddr + "/" + tpg.where + "> " +
                "[VIP: " + MCMDSocket.PMCAddr(tpg.myVirtIPAddr) + ", PIP: " + MCMDSocket.PMCAddr(tpg.myPhysIPAddr) + "] ";
            if (tpg.theView != null)
                return s + " vid " + tpg.theView.viewid + ": next mid=" + tpg.theView.viewid + ":" + tpg.nextMsgid + ", rate " + tpg.rcvdMcastsRate + ",  {" + Address.VectorToString(tpg.theView.members) + "}";
            return s + " view unknown";
        }

        // Called only in the ORACLE, to create a new Tracking Proxy
        internal static Group TrackingProxy(string gname, string where, Address gaddr, long tsig, int[] mm, View v, int flags, bool MapAddr)
        {
            Group g = new Group();
            g.isTrackingProxy = true;
            g.flags = flags & (G_ISLARGE | G_USEIPMC | G_USEUNICAST);
            g.where = where;
            g.gaddr = gaddr;
            g.gname = gname;
            g.TypeSig = tsig;
            if (mm != null && mm[MCMDSocket.VIRTUAL] != MCMDSocket.UNKNOWN)
            {
                g.myVirtIPAddr = mm[MCMDSocket.VIRTUAL];
                g.myPhysIPAddr = mm[MCMDSocket.PHYSICAL];
            }
            using (new LockAndElevate(g.ViewLock))
                g.theView = v;
            g.GroupOpen = g.WasOpen = true;
            using (new LockAndElevate(TPGroupsLock))
                TPGroups.Add(g.gaddr, g);
            if (MapAddr)
                MCMDSocket.SetMap("create TrackingProxy", gname, true, MCMDSocket.GetMap(g.gaddr, false));
            ReliableSender.StartGroupReader(g);
            return g;
        }

        internal static Group TrackingProxyLookup(string gname)
        {
            using (new LockAndElevate(TPGroupsLock))
                foreach (var kvp in TPGroups)
                {
                    if (kvp.Value.gname.Equals(gname))
                        return kvp.Value;
                }
            return null;
        }

        internal static Group TrackingProxyLookup(Address gaddr)
        {
            Group g;
            using (new LockAndElevate(TPGroupsLock))
                if (TPGroups.TryGetValue(gaddr, out g))
                    return g;
            return null;
        }

        internal static void TrackingProxyDelete(Address gaddr)
        {
            using (new LockAndElevate(TPGroupsLock))
                TPGroups.Remove(gaddr);
        }

        internal void TPGroupsLearnedMM(int[] mm)
        {
            using (new LockAndElevate(TPGroupsLock))
                foreach (var kvp in TPGroups)
                    if (kvp.Value.myVirtIPAddr == mm[MCMDSocket.VIRTUAL])
                    {
                        kvp.Value.myPhysIPAddr = mm[MCMDSocket.PHYSICAL];
                        return;
                    }
        }

        internal int[] MCMDMAP()
        {
            return new int[2] { myVirtIPAddr, myPhysIPAddr };
        }

        internal static int[] MCMDMAP(Address gaddr)
        {
            Group tpg = TrackingProxyLookup(gaddr);
            if (tpg != null)
                return tpg.MCMDMAP();
            return new int[] { MCMDSocket.UNKNOWN, MCMDSocket.UNKNOWN };
        }

        internal static List<Group> IsisGroupsClone()
        {
            List<Group> theClone = new List<Group>();
            using (new LockAndElevate(IsisGroupsLock))
                foreach (var kvp in IsisGroups)
                    theClone.Add(kvp.Value);
            return theClone;
        }

        internal static List<Group> IsisAllGroupsClone(bool removeDups)
        {
            List<Group> theClone = new List<Group>();
            using (new LockAndElevate(TPGroupsLock))
                foreach (var kvp in TPGroups)
                    theClone.Add(kvp.Value);
            using (new LockAndElevate(IsisGroupsLock))
                foreach (var kvp in IsisGroups)
                {
                    Group ig = kvp.Value;
                    bool fnd = false;
                    if (removeDups)
                        foreach (Group cg in theClone)
                            if (ig.gaddr.Equals(cg.gaddr))
                            {
                                fnd = true;
                                break;
                            }
                    if (!fnd)
                        theClone.Add(ig);
                }
            return theClone;
        }

        private void CheckState()
        {
            if (theView != null)
                throw new IsisException("Can't change attributes on an open group");
            if ((flags & (G_USEIPMC | G_USEUNICAST)) != 0)
                throw new IsisException("Can't request both useUnicast and useIPMC on same group");
        }

        /// <summary>
        /// Specifies that the user wishes this group to use the large group protocols.  
        /// </summary>
        /// <remarks>
        /// Specifies that the user wishes this group to use the large group protocols.  Recommended if a group may have 
        /// more than about 10 members, although in fact the normal protocols should be stable up to at least 100 members.
        /// </remarks>
        public void SetLarge()
        {
            CheckState();
            flags |= G_ISLARGE;
            LGSetup();
        }

        /// <summary>
        /// Group that will only be used with RawSend and other Raw IPC primitives.  No need for the "flush" virtual synchrony step
        /// </summary>
        public void SetRaw()
        {
            flags |= G_ISRAW;
        }

        /// <summary>
        /// Specifies a logging function for this group
        /// </summary>
        /// <param name="lfunc">The function</param>
        public void SetLogged(ILFunc lfunc)
        {
            myLoggingFcn = lfunc;
        }

        /// <summary>
        /// Enable or disable a per-group trace of events, which prints to the console and also to the Isis log file
        /// </summary>
        /// <param name="onOff">True to enable trace, false to disable trace</param>
        public void Trace(bool onOff)
        {
            if (onOff)
                flags |= G_TRACE;
            else
                flags &= G_TRACE;
        }

        /// <summary>
        /// Specifies that the user wishes this group to use only unicast (point to point via UDP) messaging.  
        /// </summary>
        /// <remarks>
        /// Removes this group from the ones considered for the Dr. Multicast optimal resource allocation by claiming that the received messages rate is 0
        /// </remarks>
        public void UseUnicast()
        {
            flags |= G_USEUNICAST;
            CheckState();
        }

        /// <summary>
        /// Specifies that the user wishes this group to use an IP multicast address if possible
        /// </summary>
        /// <remarks>
        /// Forces this group to the top of the sort order used in the Dr. Multicast resource allocation scheme by claiming the
        /// received message rate is infinitely high
        /// </remarks>
        public void UseIPMC()
        {
            flags |= G_USEIPMC;
            CheckState();
        }

        /// <summary>
        /// Returns a list of the failed members of the current group, if any
        /// </summary>
        /// <returns>List of members that have failed (should quickly be reported as leaving in a new view)</returns>
        public Address[] GetFailedMembers()
        {
            ConfirmJoined();
            return theView.GetFailedMembers();
        }

        /// <summary>
        /// Returns a list of the live members of the current group, normally all members of the current view
        /// </summary>
        /// <returns>List of members thatare live.  Normally this will be the full membership since new views are reported quickly. </returns>
        public Address[] GetLiveMembers()
        {
            ConfirmJoined();
            return theView.GetLiveMembers();
        }

        private void ConfirmJoined()
        {
            if (theView == null)
                throw new IsisException("Must Join before accessing membership data");
        }

        /// <summary>
        /// Returns a list of the full membership of this view, including members that have just been noted as faulty but not yet reported via a new view
        /// </summary>
        /// <returns>Membership list</returns>
        public Address[] GetMembers()
        {

            ConfirmJoined();
            return theView.members;
        }

        /// <summary>
        /// Returns the caller's rank in the group view
        /// </summary>
        /// <returns>Rank from 0..(N-1) as an offset into the full membership list of the group</returns>
        public int GetMyRank()
        {

            ConfirmJoined();
            return theView.GetMyRank();
        }

        /// <summary>
        /// Returns the rank of a designated process in the group view
        /// </summary>
        /// <param name="who"></param>
        /// <returns>Rank from 0..(N-1) as an offset into the full membership list of the group, or -1 if the process is not a group member.</returns>
        public int GetRankOf(Address who)
        {

            ConfirmJoined();
            return theView.GetRankOf(who);
        }

        /// <summary>
        /// Gets the length of the membership list
        /// </summary>
        /// <returns>N, where the membership consists of N members ranked 0..(N-1)</returns>
        public int GetSize()
        {
            ConfirmJoined();
            return theView.GetSize();
        }

        internal class SUTW : IComparable<SUTW>, IEquatable<SUTW>
        {
            // Unique ID is Sender::Uid; it "names" this SafeSend.  Current ordering is Who::TS, order is final when committed flag is true
            internal Address Sender;
            internal Address Who;
            internal int Uid;
            internal int TS;
            internal bool commitFlag = false;

            internal SUTW(Address s, int u, int t, Address w)
            {
                Sender = s; Uid = u; TS = t; Who = w;
            }

            public int CompareTo(SUTW b)
            {
                if (TS != b.TS)
                    return (TS - b.TS);
                return Who.CompareTo(b.Who);
            }

            public override int GetHashCode()
            {
                return Who.GetHashCode() | TS * 11;
            }

            public bool Equals(SUTW b)
            {
                return (TS == b.TS) && Who.Equals(b.Who);
            }
        }

        SortedList<SUTW, Msg> SSList = new SortedList<SUTW, Msg>();
        internal LockObject SSLock = new LockObject("SSLock");
        internal int logicalTS;
        internal LockObject CommitLock = new LockObject("CommitLock");
        internal LockObject ProposeLock = new LockObject("ProposeLock");

        internal class ctuple
        {
            internal int senderRank;
            internal int[] theVT;
            internal Msg theMsg;
            internal long whenEnqueued;

            internal ctuple(int sr, int[] vt, Msg m)
            {
                senderRank = sr;
                theVT = vt;
                theMsg = m;
                whenEnqueued = Isis.NOW();
            }
        }
        internal LockObject CausalOrderListLock = new LockObject("CausalOrderListLock");
        internal List<ctuple> CausalOrderList = new List<ctuple>();
        internal LockObject OrderedSubsetListLock = new LockObject("OrderedSubsetListLock");
        internal SortedList<osspq, Msg> OrderedSubsetPQ = new SortedList<osspq, Msg>();
        internal LockObject RecentOpqNodesLock = new LockObject("RecentOpqNodesLock");
        internal List<osspq> RecentOpqNodes = new List<osspq>();
        internal long myTS;

        /// <exclude></exclude>
        [AutoMarshalled]
        public class osspq : IComparable<osspq>, IEquatable<osspq>
        {
            /// <exclude></exclude>
            public bool cflag = false;
            /// <exclude></exclude>
            public long proposedTS;
            /// <exclude></exclude>
            public Address proposedWho;
            /// <exclude></exclude>
            public List<Address> dests;
            /// <exclude></exclude>
            public Address sender;
            /// <exclude></exclude>
            public int vid;
            /// <exclude></exclude>
            public int msgid;
            internal long gtime;

            internal osspq(long ts, Address who, List<Address> dests, Address sender, int vid, int msgid)
            {
                proposedTS = ts;
                proposedWho = who;
                this.dests = dests;
                this.sender = sender;
                this.vid = vid;
                this.msgid = msgid;
            }

            /// <exclude></exclude>
            public osspq()
            {
            }

            /// <exclude></exclude>
            public override string ToString()
            {
                return "OPQ[" + proposedTS + "::" + proposedWho + (cflag ? "** committed ** " : "") + ">; for msg " + sender + "::" + vid + ":" + msgid + "]";
            }

            internal void commit(long ts, Address who)
            {
                proposedTS = ts;
                proposedWho = who;
                cflag = true;
            }

            /// <exclude></exclude>
            public override bool Equals(object obj)
            {
                if (obj == null || !obj.GetType().Equals(typeof(osspq)))
                    return false;
                return Equals((osspq)obj);
            }

            /// <exclude></exclude>
            public bool Equals(osspq opq)
            {
                return sender.Equals(opq.sender) && vid == opq.vid && msgid == opq.msgid;
            }

            /// <exclude></exclude>
            public override int GetHashCode()
            {
                return vid * 771 + msgid * 33033 + sender.GetHashCode();
            }

            /// <exclude></exclude>
            public int CompareTo(object obj)
            {
                return CompareTo((osspq)obj);
            }

            /// <exclude></exclude>
            public int CompareTo(osspq opq)
            {
                if (proposedTS != opq.proposedTS)
                    return (int)(proposedTS - opq.proposedTS);
                return proposedWho.GetHashCode() - opq.proposedWho.GetHashCode();
            }
        }

        private void Setup()
        {
            // None of these procedures are permitted to block, hence we fork new threads for any that might require a long-term
            doRegister(Isis.PARTITIONED, (Action<Address>)delegate(Address who)
            {
                if (!IsisSystem.IsisRestarting)
                    throw new IsisException("Partitioning event, this node was in the minority partition");
            });

            doRegister(Isis.PROPOSE, (Action<Isis.ViewDelta[], Isis.UnstableList[]>)delegate(Isis.ViewDelta[] vds, Isis.UnstableList[] usl)
            {
                if (!HasFirstView || theView.GetMyRank() == -1)
                    throw new IsisException("Got Isis.PROPOSED in ORACLE before receiving the first ORACLE view");
                using (new LockAndElevate(ProposeLock))
                {
                    bool mustWait = false;
                    // We ddon't wedge the ORACLE since its only role is membership consensus and the scheme is our
                    // home-brew version of the leader-based paxos.  If the ORACLE had any other kind of multicasts
                    // that would be a different story, but it doesn't and we're not going to add any
                    if (this != Isis.ORACLE)
                        using (new LockAndElevate(GroupFlagsLock))
                        {
                            if ((flags & G_WEDGED) == 0)
                            {
                                mustWait = true;
                                flags |= G_WEDGED;
                            }
                        }
                    if (mustWait)
                        Wedged.WaitOne();
                    if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.VIEWWAIT)) != 0)
                    {
                        string us = " ";
                        foreach (Isis.UnstableList ul in usl)
                            us += ul.gaddr + "::" + ul.sender + ":" + ul.vid + "[" + ul.mid_low + "-" + ul.mid_hi + "] ";
                        Isis.WriteLine("=== W E D G E D < " + gname + "> vid " + theView.viewid + ", initial UNSTABLE LIST = { " + us + "} ===");
                    }
                    if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.VIEWWAIT | IsisSystem.GROUPEVENTS)) != 0)
                    {
                        foreach (Isis.ViewDelta vd in vds)
                            Isis.WriteLine("   " + vd);
                    }
                    // Creating a new thread this way may seem risky: it eliminates the risk of a deadlock associated with doing a reply
                    // but now we have to worry about event ordering relative to other incoming messages.  This is safe because the
                    // group is wedged and flushing.  Under these conditions, nothing much can happen and we can use our new thread
                    // and thus avoid running into problems like deadlock if the limit on pending asynchronous p2p messages is reached.
                    Thread t = new Thread(delegate()
                    {
                        try
                        {
                            Thread.CurrentThread.Name = "PROPOSED:DOFLUSH";
                            if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.VIEWWAIT | IsisSystem.GROUPEVENTS)) != 0)
                            {
                                foreach (Isis.ViewDelta vd in vds)
                                    Isis.WriteLine("   " + vd);
                            }
                            // This next line is safe only because there is a single dispatch thread doing both the PROPOSE and INITIALVIEW callbacks!
                            if (HasFirstView == false)
                            {
                                if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.VIEWWAIT)) != 0)
                                    Isis.WriteLine("PROPOSE handler Waiting on LLINITV barrier lock for gaddr " + gaddr + "(Lock [" + ILock.LLINITV + "][" + ILock.GetLockId(ILock.LLINITV, gaddr.GetHashCode()) + "])");
                                ILock.Barrier(ILock.LLINITV, gaddr).BarrierWait();
                                if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.VIEWWAIT)) != 0)
                                    Isis.WriteLine("PROPOSE handler After LLINITV barrier lock for gaddr " + gaddr + "(Lock [" + ILock.LLINITV + "][" + ILock.GetLockId(ILock.LLINITV, gaddr.GetHashCode()) + "])");
                            }

                            Isis.ViewDelta[] rvds = new Isis.ViewDelta[vds.Length];
                            bool involvesOracle = false, involvesMe = false;
                            Address[] leaving = new Address[0];
                            foreach (Isis.ViewDelta vd in vds)
                            {
                                if (vd.gaddr.Equals(Isis.ORACLE.gaddr))
                                    involvesOracle = true;
                                if (vd.gaddr.Equals(gaddr))
                                    involvesMe = true;
                                int len = vd.leavers.Length;
                                if (len > 0)
                                {
                                    int oldlen = leaving.Length;
                                    Isis.ArrayResize<Address>(ref leaving, oldlen + len);
                                    Array.Copy(vd.leavers, 0, leaving, oldlen, len);
                                }
                            }
                            Isis.Proposed = vds;

                            if (involvesMe == false || (this == Isis.ORACLE && involvesOracle == false))
                            {
                                // Special case: Oracle was "cc'ed" and should ack, but has no actual role
                                if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                                    Isis.WriteLine("PROPOSE: doReply special case (ORACLE acks)");
                                doReply(Isis.my_address, new Isis.ViewDelta[0]);
                                goto done;
                            }

                            // Give the application a chance to flush, if using the new FlushHandler upcalls
                            using (new LockAndElevate(FlushHandlers.vhListLock))
                                if (FlushHandlers.vhList.Count() > 0)
                                {
                                    View v;
                                    using (new LockAndElevate(ViewLock))
                                        v = new View(theView.gname, theView.gaddr, theView.members, theView.viewid, theView.isLarge);
                                    // Tinker with the View to report who will join or leave without actually modifying the list of members
                                    foreach (Isis.ViewDelta vd in vds)
                                        if (vd.gaddr.Equals(gaddr) && vd.prevVid == v.viewid)
                                        {
                                            Isis.UpdateGroupView(false, vd, this, "PROPOSE:FLushUpcall", ref v, true);
                                            foreach (VHCallBack vhcb in FlushHandlers.vhList)
                                                vhcb.vhProc(v);
                                        }
                                }

                            // Flush those I've been asked to forward.  Code takes advantage of the group being in a WEDGED state: no new traffic allowed in...
                            using (Semaphore CPSSema = new Semaphore(0, int.MaxValue))
                            {
                                int cpscnt;
                                List<Msg> mustSend;
                                startFlush(int.MaxValue, usl, out mustSend, out cpscnt, CPSSema);
                                endFlush(leaving, mustSend, cpscnt, CPSSema);
                            }

                            // Now look up the groups, again, and this time note the final Msgids
                            // for those vds corresponding to "my" group
                            for (int idx = 0; idx < vds.Length; idx++)
                            {
                                Isis.ViewDelta vd = vds[idx];
                                rvds[idx] = new Isis.ViewDelta(vd.gname, vd.gaddr, 0L, Group.MCMDMAP(vd.gaddr), -1, new int[0], false);
                                if (vd.gaddr.Equals(gaddr) == false)
                                    continue;
                                if ((flags & G_ISLARGE) != 0)
                                    throw new IsisException("PROPOSE: vd applied to a LARGE group!");

                                if (this != Isis.ORACLE || IAmLeader() == false)
                                    theView.isFinal = true;
                                rvds[idx].prevVid = theView.viewid;
                                rvds[idx].lastSeqns = new int[theView.NextIncomingMsgID.Length - 1];
                                string ls = " ";
                                for (int i = 0; i < theView.NextIncomingMsgID.Length - 1; i++)
                                {
                                    ls += theView.NextIncomingMsgID[i + 1] + " ";
                                    rvds[idx].lastSeqns[i] = theView.NextIncomingMsgID[i + 1];
                                }
                                foreach (Address d in vd.leavers)
                                {
                                    int r = theView.GetRankOf(d);
                                    if (r != -1)
                                        View.noteFailed(this, d);
                                }
                            }

                            if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                                Isis.WriteLine("PROPOSE: doReply normal case (processed " + rvds.Length + " view deltas), curMsg=" + getReplyTo());
                            if (this == Isis.ORACLE)
                                doReply(Isis.my_address, new Isis.ViewDelta[0]);
                            else
                                doReply(Isis.my_address, rvds);
                        }
                        catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    done:
                        IsisSystem.ThreadTerminationMagic();
                    });
                    SetReplyThread(t);
                    t.Priority = ThreadPriority.Highest;
                    t.Start();
                }
            });

            doRegister(Isis.COMMIT, (Action<Isis.ViewDelta[], Address[], int[]>)delegate(Isis.ViewDelta[] vds, Address[] who, int[] uid)
            {
                if (!HasFirstView || theView.GetMyRank() == -1)
                    throw new IsisException("Got Isis.PROPOSED in ORACLE before receiving the first ORACLE view");
                using (new LockAndElevate(groupLock))
                using (new LockAndElevate(CommitLock))
                {
                    if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                    {
                        string s = "Received a new COMMIT message in group <" + gname + "> ViewDelta vector has " + vds.Length + " updates:\r\n";
                        foreach (Isis.ViewDelta vd in vds)
                            s += "  " + vd + "\r\n";
                        Isis.Write(s);
                    }
                    Isis.ViewDelta[] nextvds = null;
                    do
                    {
                        if (this == Isis.ORACLE)
                        {
                            // Look at "chunks" of the vds vector that end with an ORACLE join event.
                            int n;
                            for (n = 0; n < vds.Length - 1; n++)
                                if (vds[n].gaddr.Equals(Isis.ORACLE.gaddr) && vds[n].joiners.Length > 0)
                                    break;
                            ++n;  // n is now the length of the chunk
                            if (n < vds.Length)
                            {
                                nextvds = new Isis.ViewDelta[vds.Length - n];
                                for (int i = n; i < vds.Length; i++)
                                    nextvds[i - n] = vds[i];
                                Isis.ArrayResize<Isis.ViewDelta>(ref vds, n);
                            }
                            else
                                nextvds = null;

                            // Now there is a chunk in vds, and perhaps more work to do later in nextvds
                            // Ignore the current chunk if I've already seen it, or it predates my joining the ORACLE
                            if ((n = vds.Length - 1) < 0)
                                return;
                            if (vds[n].gaddr.Equals(Isis.ORACLE.gaddr) && vds[n].joiners.Length > 0)
                                using (new LockAndElevate(ViewLock))
                                    if (theView != null)
                                        if (vds[n].prevVid < theView.viewid)
                                        {
                                            if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                                            {
                                                string s = "IGNORING STALE COMMIT actions in group <" + gname + "> (theView.viewid=" + theView.viewid + "), Ignored part of ViewDelta vector has " + vds.Length + " updates:";
                                                foreach (Isis.ViewDelta vd in vds)
                                                    s += "  " + vd + "\r\n";
                                                Isis.Write(s);
                                            }
                                            continue;
                                        }
                        }
                        if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                        {
                            string s = "APPLYING COMMIT actions in group <" + gname + "> (theView.viewid=" + theView.viewid + "), this chunk of ViewDelta vector has " + vds.Length + " updates:";
                            foreach (Isis.ViewDelta vd in vds)
                                s += "  " + vd + "\r\n";
                            Isis.Write(s);
                        }
                        if (this != Isis.ORACLE)
                        {
                            int myVid = 0;
                            using (new LockAndElevate(ViewLock))
                                if (theView != null)
                                    myVid = theView.viewid;
                            vds = vds.Where(vd => vd.gaddr.Equals(this.gaddr) && vd.prevVid >= myVid).ToArray();
                            if (vds.Count() == 0)
                                return;
                        }
                        gotNewViewDeltas(vds);
                        Isis.CommitGVUpdates(this, vds);
                        if (this == Isis.ORACLE)
                            for (int i = 0; i < who.Length; i++)
                                Isis.PurgeGVE(who[i], uid[i]);
                        Isis.Proposed = null;
                        if (this == Isis.ORACLE)
                        {
                            using (new LockAndElevate(Isis.ORACLE.groupLock))
                            {
                                foreach (Isis.ViewDelta vd in vds)
                                {
                                    if (vd.gaddr.Equals(Isis.ORACLE.gaddr))
                                        Isis.OracleJoinsUnderway -= vd.joiners.Length;
                                }
                                if (Isis.OracleJoinsUnderway < 0)
                                    Isis.OracleJoinsUnderway = 0;
                            }
                            if (Isis.ORACLE.IAmLeader())
                                ILock.Barrier(ILock.LLWAIT, ILock.LCOMMIT).BarrierRelease(1);
                            Isis.SendInitialView(vds);
                        }
                        if (this != Isis.ORACLE)
                            using (new LockAndElevate(GroupFlagsLock))
                                if ((flags & G_WEDGED) != 0)
                                {
                                    Wedged.Release();
                                    flags &= ~G_WEDGED;
                                }
                    }
                    while ((vds = nextvds) != null);
                }
                List<Address> ripdup = new List<Address>();
                using (new LockAndElevate(Isis.RIPLock))
                    foreach (Address rip in Isis.RIPList)
                        ripdup.Add(rip);
                foreach (Address rip in ripdup)
                {
                    int rank = theView.GetRankOf(rip);
                    if (rank != -1)
                        View.noteFailed(this, rip);
                }
                if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                    Isis.WriteLine("Done with the COMMIT message in group  " + gname);
                if (theView.GetMyRank() == -1)
                    GroupClose();
            });

            doRegister(Isis.COMMIT, (Action<Isis.ViewDelta[]>)delegate(Isis.ViewDelta[] vds)
            {
                View theView = null;
                using (new LockAndElevate(ViewLock))
                    theView = this.theView;
                vds = vds.Select(vd => vd).Where(vd => vd.gaddr.Equals(this.gaddr) && vd.prevVid >= (theView == null ? -1 : theView.viewid)).ToArray();
                if (vds.Length == 0)
                    return;
                int initialSeqn = curMsgId();
                if ((IsisSystem.Debug & (IsisSystem.GROUPEVENTS | IsisSystem.TOKENLOGIC)) != 0)
                {
                    Isis.WriteLine("Received a new LGCOMMIT message in group <" + gname + "> ViewDelta vector has " + vds.Length + " updates:");
                    foreach (Isis.ViewDelta vd in vds)
                        Isis.WriteLine("  " + vd);
                }
                if (vds.Length > 0)
                    gotNewViewDeltas(vds);
                if (theToken != null)
                {
                    theToken.logicalClock++;
                    theToken.resetStableByLevel(this);
                }
                using (new LockAndElevate(CommitLock))
                    Isis.CommitGVUpdates(this, vds, ref theView);
                foreach (Isis.ViewDelta vd in vds)
                    if (vd.joiners.Length > 0)
                    {
                        if (vd.prevVid == -1)
                        {
                            Isis.WriteLine("Calling GetMap(" + gname + ", true) in Isis.COMMIT");
                            MCMDSocket.GetMap(gaddr, true);
                        }
                        if (IAmRank0())
                        {
                            Address[] AllJoiners = Isis.Expand(vd.joiners);
                            foreach (Address who in AllJoiners)
                            {
                                byte[] ba = Msg.toBArray(Isis.INITIALVIEW, Isis.my_address, theView, MCMDSocket.GetMap(gaddr, false), initialSeqn);
                                if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.TOKENLOGIC)) != 0)
                                    Isis.WriteLine("LargeGroup owner for <" + gname + "> sending INITIALVIEW with initialSeqn=" + initialSeqn + " to " + who + "(view = " + theView + ")");
                                ReliableSender.SendP2P(Msg.ISGRPP2P, who, this, theView.viewid, ReliableSender.P2PSequencer.NextP2PSeqn("initialview/lg", who), ba, true, null, null);
                            }
                        }
                    }
                Isis.Proposed = null;
                if ((IsisSystem.Debug & (IsisSystem.GROUPEVENTS | IsisSystem.TOKENLOGIC)) != 0)
                    Isis.WriteLine("Done with the LGCOMMIT message in group  " + gname);
            });

            doRegister(Isis.INQUIRE, (Action<Address>)delegate(Address newLeader)
            {
                // This runs only in the ORACLE and uses a separate thread just because it sends a reply and we were worried about
                // some obscure deadlock risks involving waiting because of too many asynchronous p2p messages in the reliable sender
                // subsystem.  In fact the ORACLE only runs the leader-based 2pc/3pc protocol (this is the 1st phase message in the 3pc case
                // when a new leader takes over) and the group is otherwise "idle", so whether we use a new thread or not makes
                // no difference to event ordering
                Thread t = new Thread(delegate()
                {
                    try
                    {
                        Thread.CurrentThread.Name = "Receive INQUIRE about existing known view updates";
                        if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                            Isis.WriteLine("Received a new INQUIRE request in group  " + gname + "  from " + newLeader);
                        int hisRank = theView.GetRankOf(newLeader);
                        if (hisRank == -1) throw new IsisException("INQUIRE from a new leader who isn't in the group (or isn't alive)!" + IsisSystem.GetState());
                        for (int i = 0; i < hisRank - 1; i++)
                            if (theView.hasFailed[i] == false)
                                Isis.NodeHasFailed(theView.members[i], "(from INQUIRE)", true);
                        Isis.ViewDelta[] proposed = Isis.Proposed;
                        if (proposed == null)
                            proposed = new Isis.ViewDelta[0];
                        if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                            Isis.WriteLine("Reply to INQUIRE request in group  " + gname + "  with " + proposed.Length + " events");
                        doReply(Isis.my_address, Isis.ORACLE.theView.members[0], proposed);
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    IsisSystem.ThreadTerminationMagic();
                });
                SetReplyThread(t);
                t.Start();
            });

            doRegister(Isis.INITIALVIEW, (Action<Address, string[], Address[], long[], View[], bool[], int, int[,], Address[]>)delegate(Address sender, string[] names, Address[] gaddrs, long[] tsigs, View[] vs, bool[] isl, int curPhysIPA, int[,] mms, Address[] CBMs)
            {
                IsisSystem.IsisRestarting = false;
                if ((IsisSystem.Debug & (IsisSystem.GROUPEVENTS | IsisSystem.STARTSEQ | IsisSystem.MCMDMAP)) != 0)
                {
                    Isis.WriteLine("Received a new ORACLE-member INITIALVIEW event in group " + gname + ", and Isis.ClientOf is " + (Isis.ClientOf == null ? "null" : "non-null (" + Isis.ClientOf + ")"));
                    for (int i = 0; i < names.Length; i++)
                        Isis.WriteLine("   .... <" + names[i] + ">: " + (isl[i] ? "(long) " : "") + gaddrs[i] + ", View=" + vs[i]);
                    Isis.WriteLine("MCMD Mapping information:");
                    for (int i = 0; i < mms.GetLength(0); i++)
                        Isis.WriteLine("    Map virtual address " + MCMDSocket.PMCAddr(mms[i, MCMDSocket.VIRTUAL]) + " to " + MCMDSocket.PMCAddr(mms[i, MCMDSocket.PHYSICAL]));
                }
                if (Isis.ClientOf != null)
                    throw new IsisException("ORACLE initial-view event unexpected when in client-of mode");
                using (new LockAndElevate(Isis.CanBeOracleListLock))
                    foreach (Address a in CBMs)
                        Isis.CanBeOracleList.Add(a);
                MCMDSocket.nextPhysIPAddr = curPhysIPA;
                for (int i = 0; i < names.Length; i++)
                {
                    if (names[i].Equals("ORACLE"))
                    {
                        Group g = Isis.ORACLE;
                        Isis.LeaderId = vs[i].leaderId;
                        if (g.HasFirstView == false || g.theView.viewid < vs[i].viewid)
                        {
                            g.NewView(vs[i], "ORACLE", new int[2] { mms[i, MCMDSocket.VIRTUAL], mms[i, MCMDSocket.PHYSICAL] });
                            g.ReplayToDo();
                        }
                        if (g.theView.GetRankOf(Isis.my_address) == -1)
                            throw new IsisException("ORACLE initial-view GROUP LIST: oracle didn't list me\r\n" + IsisSystem.GetState());
                    }
                    else
                    {
                        Group tpg = Group.TrackingProxyLookup(gaddrs[i]);
                        if (tpg == null)
                            tpg = Group.TrackingProxy(names[i], "INITIALVIEW[]", gaddrs[i], tsigs[i], null, vs[i], isl[i] ? G_ISLARGE : 0, false);
                        if (tpg != null && (tpg.HasFirstView == false || tpg.theView.members.Length == 0 || tpg.theView.viewid <= vs[i].viewid))
                            tpg.NewView(vs[i], "view[] initializer", new int[2] { mms[i, MCMDSocket.VIRTUAL], mms[i, MCMDSocket.PHYSICAL] });
                        if (!tpg.hasPhysMapping && mms[i, MCMDSocket.PHYSICAL] != MCMDSocket.UNKNOWN)
                        {
                            tpg.hasPhysMapping = true;
                            MCMDSocket.SetMap("ORACLE initializer", tpg.gname, true, new int[2] { mms[i, MCMDSocket.VIRTUAL], mms[i, MCMDSocket.PHYSICAL] });
                        }
                    }
                }
                MCMDSocket.AssignMapInfo(mms);
                if ((IsisSystem.Debug & (IsisSystem.GROUPEVENTS | IsisSystem.STARTSEQ | IsisSystem.MCMDMAP)) != 0)
                    Isis.WriteLine("Finished in ORACLE-member Isis.INITIALVIEW: my state is " + IsisSystem.GetState());
            });

            doRegister(Isis.INITIALVIEW, (Action<Address, View, int[], int>)delegate(Address sender, View v, int[] mm, int initialseqn)
            {
                AcceptInitialView(sender, v, mm, initialseqn);
            });

            doRegister(Isis.INITIALVIEW, (Action<Address, string, int[], int>)delegate(Address sender, string OOBFname, int[] mm, int initialseqn)
            {
                Thread t = new Thread(delegate()
                {
                    MemoryMappedFile mmf = OOBFetch(OOBFname);
                    if (mmf != null)
                    {
                        MemoryMappedViewAccessor mmva = mmf.CreateViewAccessor();
                        byte[] ba = new byte[mmva.Capacity];
                        mmva.ReadArray<byte>(0, ba, 0, (int)mmva.Capacity);
                        AcceptInitialView(sender, new View(ba), mm, initialseqn);
                    }
                    else
                        throw new IsisException("OOBFetch failed for " + OOBFname);
                });
                t.Name = "Fetch INITIALVIEW";
                t.Start();
            });

            doRegister(Isis.JOINFAILED, (Action<Address, string>)delegate(Address gaddr, string reason)
            {
                Group g = Group.doLookup(gaddr);
                g.joinFailed = true;
                g.reason = reason;
                ILock.Barrier(ILock.LLINITV, gaddr).BarrierReleaseAll();
            });

            doRegister(Isis.INITIALVIEW, (Action<Address>)delegate(Address sender)
            {
                if ((IsisSystem.Debug & (IsisSystem.GROUPEVENTS | IsisSystem.STARTSEQ)) != 0)
                    Isis.WriteLine("Received a new client-of INITIALVIEW event in group " + gname + ", and Isis.ClientOf is " + (Isis.ClientOf == null ? "null" : "non-null (" + Isis.ClientOf + ")") + " and <ORACLE>=" + Isis.ORACLE.ToString());
                if (sender.isMyAddress())
                    throw new IsisException("Client-of myself!\r\n" + IsisSystem.GetState());
                if (Isis.ORACLE.theView != null && Isis.ORACLE.theView.GetMyRank() != -1)
                {
                    Isis.WriteLine("ORACLE owner told me to be a client, then changed his mind (WARNING only)");
                    return;
                }
                IsisSystem.IsisRestarting = false;
                Group g;
                if ((g = Group.doLookup("ORACLE")) != null && g.HasFirstView)
                    g.Leave();
                else if (g != null)
                    g.GroupClose();
                Isis.ClientOf = sender;
                Isis.OracleFailedAt = 0;
                Isis.ORACLE = TrackingProxy("ORACLE", "Client-Of initializer", Isis.ORACLE.gaddr, 0, null, new View("ORACLE", Isis.ORACLE.gaddr, new Address[] { sender }, 0, (flags & G_ISLARGE) != 0), flags, false);
                ILock.Barrier(ILock.LLWAIT, ILock.LCLIENTOF).BarrierRelease(1);
                if ((IsisSystem.Debug & IsisSystem.STARTSEQ) != 0)
                    Isis.WriteLine("Finished in Isis.INITIALVIEW (client-of case): my state is " + IsisSystem.GetState());
            });

            doRegister(Isis.STATEXFER, (Action)delegate()
            {
                if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                    Isis.WriteLine("Received end-of-statexfer (EndXFER) in group " + gname);
                EndStateXfer();
            });

            doRegister(Isis.CRYPTOWRAPPED, (Action<byte[]>)delegate(byte[] ciphered)
            {
                // A crypto-wrapped message is a p2p message with an enciphered body.  Decrypt it, then redeliver using the "true" body
                if (myAes == null)
                    throw new IsisException("Received an enciphered object but this group member didn't know the key!");
                byte[] buffer = decipherBuf(ciphered);
                if (buffer != null && buffer.Length > 0)
                    Redeliver(buffer);
                else
                    using (new LockAndElevate(GroupFlagsLock))
                        if ((flags & G_NEEDSTATEXFER) != 0)
                            EndStateXfer();
            });
            doRegister(Isis.CLIENTWRAPPED, (Action<string, int, byte[]>)delegate(string gname, int rcode, byte[] clientreq)
            {
                // A client-wrapped message is one sent from the Isis "Client of a group" API; it arrives in Isis.ISISMEMBERS but needs to be redelivered in the
                // actual target group, which involves moving the true "reply-to" message from ISISMEMBERS to the target group, and also getting the redeliver API
                // to check for redelivery information on the redelivery queue of ISISMEMBERS rather than the one associated with the target group
                Group g = Group.Lookup(gname);
                if (g != null)
                {
                    Msg rmsg = getReplyToAndClear();
                    if (rmsg != null)
                    {
                        rmsg.gaddr = gaddr;
                        g.setReplyTo(rmsg);
                    }
                    if (g.allowsClientRequests[rcode + Isis.SYSTEMREQS])
                        g.Redeliver(clientreq, this.rdiLock, this.rdiList);
                    else
                        g.AbortReply("Group has not enabled client requests to request code " + rcode + " (did you forget to call g.AllowClientRequests?)");
                }
                else
                    AbortReply("Group <" + gname + "> not found");
            });

            doRegister(Isis.FRAGMENT, (Action<Address, int, long, int, int, bool, byte[]>)delegate(Address sender, int fid, long tl, int nf, int fn, bool ic, byte[] frag)
            {
                deFragGotFrag(this, sender, fid, tl, nf, fn, ic, frag);
            });

            doRegister(Isis.REMAP, (Action<int, MCMDSocket.GRPair[]>)delegate(int id, MCMDSocket.GRPair[] grps)
            {
                if ((IsisSystem.Debug & IsisSystem.MCMDMAP) != 0)
                    Isis.WriteLine("Recomputing MCMD Mapping!");
                MCMDSocket.ComputeMCMDMapping(id, grps);
            });

            doRegister(Isis.REMAP, (Action<int, int, int[,]>)delegate(int epochId, int nip, int[,] ma)
            {
                MCMDSocket.nextPhysIPAddr = nip;
                Isis.MapperEpochId = Math.Max(Isis.MapperEpochId, epochId + 1);
                MCMDSocket.ReMap(epochId, ma);
            });

            doRegister(Isis.REMAP, (Action<int[,]>)delegate(int[,] ma)
            {
                using (new LockAndElevate(Group.IsisGroupsLock))
                    foreach (var kvp in IsisGroups)
                    {
                        Group g = kvp.Value;
                        int virtIPAddr = g.myVirtIPAddr;
                        if (virtIPAddr == MCMDSocket.UNKNOWN)
                            virtIPAddr = g.myVirtIPAddr = Isis.CLASSD + Isis.ISIS_MCRANGE_LOW + Address.GroupNameHash(g.gname);
                        if (g.myPhysIPAddr == MCMDSocket.UNKNOWN)
                            for (int i = 0; i < ma.GetLength(0); i++)
                                if (ma[i, MCMDSocket.VIRTUAL] == virtIPAddr)
                                {
                                    g.myVirtIPAddr = virtIPAddr;
                                    g.myPhysIPAddr = ma[i, MCMDSocket.PHYSICAL];
                                }
                    }
            });

            doRegister(Isis.TERMINATE, (Action)delegate()
            {
                if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                    Isis.WriteLine("Group <" + gname + "> received TERMINATE event" + IsisSystem.GetState());
                if ((flags & G_ISLARGE) != 0 && !TermSent && IAmLeader())
                {
                    TermSent = true;
                    doSendNotFromOracle(Isis.TERMINATE);
                }
                GroupClose();
            });

            doRegister(Isis.TERMINATE, (Action<Address>)delegate(Address gaddr)
            {
                PrepareForTermination();
                doNullReply();
            });

            doRegister(Isis.FANNOUNCE, (Action<Address>)delegate(Address which)
            {
                ReliableSender.NodeHasFailed(which);
                GroupNoteFailure(this, which);
            });

            doRegister(Isis.OUTOFBAND, (Action<string, Address, List<Address>>)delegate(string OOBFname, Address gaddr, List<Address> where)
            {
                doOOBUpdateRegistry(OOBFname, gaddr, where);
            });

            doRegister(Isis.OUTOFBAND, (Action<List<OOBRepInfo>>)delegate(List<OOBRepInfo> repInfo)
            {
                doOOBUpdateRegistry(repInfo);
            });

            doRegister(Isis.OUTOFBAND, (Action<string, long, byte[]>)delegate(string OOBFname, long off, byte[] data)
            {
                doOOBGotChunk(OOBFname, off, data);
            });

            doRegister(Isis.OUTOFBAND, (Action<string, string, Address, int, long, List<Address>>)delegate(string mcaddr, string OOBFname, Address gaddr, int nchunks, long len, List<Address> where)
            {
                doOOBPrepare(mcaddr, OOBFname, gaddr, nchunks, len, where);
            });

            doRegister(Isis.OUTOFBAND, (Action<string, int>)delegate(string OOBFname, int nc)
            {
                doReportChunkStatus(OOBFname, nc);
            });

            doRegister(Isis.OUTOFBAND, (Action<string>)delegate(string OOBFname)
            {
                doOOBDone(OOBFname);
            });

            doRegister(Isis.CAUSALSEND, (Action<int, int[], Msg>)delegate(int vid, int[] theVT, Msg theMsg)
            {
                View theView;
                using (new LockAndElevate(ViewLock))
                    theView = this.theView;
                if (vid != theView.viewid)
                {
                    // If CausalSend shows up in a different view than the one in which it was sent, the prior messages will
                    // have been delivered by virtue of the group flush protocol having run to completion during the view change
                    incomingSends.put(new Msg(theMsg));
                    return;
                }
                int senderRank;
                Msg cm = curMsg();
                if (cm == null)
                    throw new IsisException("CausalSend: curMsg null");
                else
                    senderRank = theView.GetRankOf(cm.sender);
                if ((IsisSystem.Debug & IsisSystem.CAUSALDELIVERY) != 0)
                    Isis.WriteLine("Got new CausalSend with viewid=" + vid + ", VT=" + VTtoString(theVT) + " from sender with rank " + senderRank + ", my VT=" + VTtoString(theView.myVT));
                List<ctuple> newCList = new List<ctuple>();
                List<ctuple> toDeliver = new List<ctuple>();
                ctuple myCT = new ctuple(senderRank, theVT, theMsg);
                using (new LockAndElevate(CausalOrderListLock))
                {
                    bool inserted = false;
                    foreach (ctuple ct in CausalOrderList)
                    {
                        if (myCT != null && happensBefore(theVT, ct.theVT))
                        {
                            inserted = true;
                            newCList.Add(myCT);
                        }
                        newCList.Add(ct);
                    }
                    if (!inserted)
                        newCList.Add(myCT);
                    CausalOrderList = newCList;
                    bool fnd;
                    do
                    {
                        fnd = false;
                        List<ctuple> oldClist = CausalOrderList;
                        CausalOrderList = new List<ctuple>();
                        foreach (ctuple ct in oldClist)
                            if (isDeliverable(ct.senderRank, ct.theVT))
                            {
                                fnd = true;
                                if ((IsisSystem.Debug & IsisSystem.CAUSALDELIVERY) != 0)
                                    Isis.WriteLine("Causal delivery with viewid=" + theView.viewid + ", VT=" + VTtoString(ct.theVT) + " of " + ct.theMsg + ", my VT=" + VTtoString(theView.myVT));
                                toDeliver.Add(ct);
                            }
                            else
                                CausalOrderList.Add(ct);
                    }
                    while (fnd);
                }
                bool dme = false;
                foreach (ctuple ct in toDeliver)
                {
                    if (ct == myCT)
                        dme = true;
                    incomingSends.put(new Msg(ct.theMsg));
                }
                if (!dme && (IsisSystem.Debug & IsisSystem.CAUSALDELIVERY) != 0)
                    Isis.WriteLine("Needed to delay CausalSend with viewid=" + vid + ", VT=" + VTtoString(theVT) + " from sender with rank " + senderRank + ", my VT=" + VTtoString(theView.myVT));
            });

            doRegister(Isis.ORDEREDSEND, (Action<Msg>)delegate(Msg theMsg)
            {
                if ((IsisSystem.Debug & IsisSystem.ORDEREDSEND) != 0)
                    Isis.WriteLine("Isis.ORDEREDSEND[" + curMsg().sender + "::" + curMsg().vid + ":" + curMsg().msgid + "]: Request to establish ordering for  " + theMsg.sender + "::" + theMsg.vid + ":" + theMsg.msgid);
                using (new LockAndElevate(OutOfOrderQueueLock))
                    OutOfOrderQueue.Add(theMsg);
                if (theView.IAmLeader() && !onDOQ(theMsg) && (flags & G_WEDGED) == 0)
                {
                    Address[] senders;
                    int[] vids, msgids;
                    GenerateOrdering(out senders, out vids, out msgids, false);
                    // By using Send instead of doSend, this code deliberately allows the Send to be blocked if the group
                    // is wedged for a membership change (doSend ignores that kind of wedging).  In such cases upon receipt,
                    // the corresponding message will already have been delivered. But this also forces creation of a new task
                    // to avoid blocking the group's multicast delivery thread
                    if ((IsisSystem.Debug & IsisSystem.ORDEREDSEND) != 0)
                        Isis.WriteLine("Isis.ORDEREDSEND: GeneratedOrdering<" + gname + ">=" + OrderToString(senders, vids, msgids));
                    if (senders.Length > 0)
                        NonFlowControlledSend(Isis.SETORDER, Isis.my_address, senders, vids, msgids);
                }
                else
                    // This may be what we were waiting for, deliver whatever we now can deliver without a new SETORDER
                    DeliverInOrder("Isis.ORDEREDSEND(nulls)", new Address[0], new int[0], new int[0]);
            });

            // These versions are for the Ordered subset multicast.
            doRegister(Isis.ORDEREDSEND, (Action<List<Address>, long, Msg>)delegate(List<Address> dests, long hisTS, Msg m)
            {
                osspq opq;
                if ((IsisSystem.Debug & IsisSystem.ORDEREDSEND) != 0)
                    Isis.WriteLine("ORDEREDSEND==> Got dests=" + Address.VectorToString(dests.ToArray()) + ", id=" + m.sender + "::" + m.vid + ":" + m.msgid + ", m=" + m);
                using (new LockAndElevate(OrderedSubsetListLock))
                {
                    if (hisTS > myTS)
                        myTS = hisTS;
                    opq = new osspq(++myTS, Isis.my_address, dests, m.sender, m.vid, m.msgid);
                    OrderedSubsetPQ.Add(opq, m);
                }
                if ((IsisSystem.Debug & IsisSystem.ORDEREDSEND) != 0)
                    Isis.WriteLine("ORDEREDSEND==> doReply(" + opq.proposedTS + "::" + opq.proposedWho + ")");
                doReply(opq.proposedTS, opq.proposedWho);
            });

            // Learned the commit time.  No need for locks: this runs only on the P2P delivery thread
            doRegister(Isis.ORDEREDSEND, (Action<Address, int, int, long, Address>)delegate(Address sentBy, int vid, int msgid, long cts, Address ctwho)
            {
                if ((IsisSystem.Debug & IsisSystem.ORDEREDSEND) != 0)
                    Isis.WriteLine("ORDEREDSEND==>COMMIT (" + sentBy + "::" + vid + ":" + msgid + ") at (" + cts + "::" + ctwho + ")");
                opqDeliver(sentBy, vid, msgid, cts, ctwho);
            });

            doRegister(Isis.ORDEREDSEND, (Action<List<osspq>>)delegate(List<osspq> recentOpqNds)
            {
                if ((IsisSystem.Debug & IsisSystem.ORDEREDSEND) != 0)
                {
                    foreach (osspq opq in recentOpqNds)
                        Isis.WriteLine("ORDEREDSEND==>COMMIT (" + opq.sender + "::" + opq.vid + ":" + opq.msgid + ") at (" + opq.proposedTS + "::" + opq.proposedWho + ")");
                    Isis.WriteLine("ORDEREDSEND==>got flush finalizer containing");
                }
                foreach (osspq opq in recentOpqNds)
                    opqDeliver(opq.sender, opq.vid, opq.msgid, opq.proposedTS, opq.proposedWho);
            });

            FlushHandlers += (ViewHandler)delegate(View v)
            {
                Dictionary<Address, List<osspq>> toSend = new Dictionary<Address, List<osspq>>(100);
                using (new LockAndElevate(RecentOpqNodesLock))
                    foreach (osspq opq in RecentOpqNodes)
                    {
                        if (!opq.cflag)
                            throw new IsisException("Found an uncommitted opq node on the RecentOpqNodes list");
                        List<osspq> theList;
                        if (toSend.TryGetValue(opq.sender, out theList) == false)
                            toSend.Add(opq.sender, new List<osspq> { opq });
                        else
                            theList.Add(opq);
                    }
                foreach (var kvp in toSend)
                    P2PSend(kvp.Key, Isis.ORDEREDSEND, kvp.Value);
                Flush();
            };

            doRegister(Isis.SETORDER, (Action<Address, Address[], int[], int[]>)delegate(Address orderedBy, Address[] senders, int[] vids, int[] msgids)
            {
                if ((IsisSystem.Debug & IsisSystem.ORDEREDSEND) != 0)
                {
                    Msg theMsg = curMsg();
                    Isis.WriteLine("Isis.SETORDER: Received " + theMsg.sender + "::" + theMsg.vid + ":" + theMsg.msgid + ", containing " + OrderToString(senders, vids, msgids));
                }
                DeliverInOrder("Isis.SETORDER", senders, vids, msgids);
            });

            doRegister(Isis.SAFESEND, (Action<Address, int, Msg>)delegate(Address sender, int Uid, Msg sMsg)
            {
                int myTS;
                using (new LockAndElevate(SSLock))
                {
                    myTS = ++logicalTS;
                    SSList.Add(new SUTW(sender, Uid, myTS, Isis.my_address), sMsg);
                }
                if ((IsisSystem.Debug & IsisSystem.SAFESEND) != 0)
                    Isis.WriteLine("Isis.SAFESEND: Got " + sender + "::" + Uid + " for pending message " + sMsg.sender + "::" + sMsg.vid + ":" + sMsg.msgid + ", my proposal: " + Isis.my_address + "::" + myTS);
                if (durabilityMethod != null)
                    sMsg.ct = durabilityMethod.LogMsg(sMsg);
                if (safeSendThreshold == ALL || theView.GetMyRank() < safeSendThreshold)
                    doReply(myTS, Isis.my_address);
                else
                    NoReply();
            });

            doRegister(Isis.SAFEDELIVER, (Action<Address, int, int, Address>)delegate(Address sender, int Uid, int TS, Address who)
            {
                if ((IsisSystem.Debug & IsisSystem.SAFESEND) != 0)
                    Isis.WriteLine("Isis.SAFEDELIVER: Got final TS " + who + "::" + TS + " for pending message " + sender + "::" + Uid);
                bool fnd = false;
                using (new LockAndElevate(SSLock))
                {
                    foreach (KeyValuePair<SUTW, Msg> kvp in SSList)
                    {
                        SUTW suw = kvp.Key;
                        Msg sMsg = kvp.Value;
                        if (suw.Sender.Equals(sender) && suw.Uid == Uid)
                        {
                            SSList.Remove(suw);
                            suw.TS = TS;
                            suw.Who = who;
                            suw.commitFlag = true;
                            SSList.Add(suw, sMsg);
                            if (TS > logicalTS)
                                logicalTS = TS;
                            fnd = true;
                            break;
                        }
                    }
                    if (fnd)
                        deliverSSItems();
                }
            });

            doRegister(Isis.LOCKREQ, (Action<int, string, Address, int>)delegate(int action, string lockName, Address who, int cntr)
            {
                LockInfo lstate;
                bool grantIt = false, cancelIt = false;
                using (new LockAndElevate(LocksListLock))
                {
                    if (!LocksList.TryGetValue(lockName, out lstate))
                    {
                        lstate = new LockInfo(lockName);
                        LocksList[lockName] = lstate;
                    }
                    Msg cm = curMsg();
                    string minfo = "[" + (cm == null ? "<<null msg>>" : (cm.sender + "::" + cm.vid + ":" + cm.msgid)) + "]";
                    if (cm.vid != lockCurrentViewid)
                        throw new IsisException(minfo + ": ERROR in LOCKREQ -- message has vid " + cm.vid + ", but LockViewId = " + lockCurrentViewid);
                    switch (action)
                    {
                        case LOCKIT:
                            if ((IsisSystem.Debug & IsisSystem.LOCKS) != 0)
                                Isis.WriteLine(minfo + "; LOCKIT<" + cntr / 10000 + "." + cntr % 1000 + ">: " + lockName + ", who=" + who);
                            if (lstate.wantLock.Contains(who))
                                throw new IsisException("LOCKIT: wantLock already contained " + who);
                            lstate.wantLock.Add(who);
                            if (!lstate.islocked)
                                grantIt = true;
                            break;
                        case RELEASEIT:
                            if ((IsisSystem.Debug & IsisSystem.LOCKS) != 0)
                                Isis.WriteLine(minfo + "; RELEASEIT<" + cntr / 10000 + "." + cntr % 1000 + ">: " + lockName + ", who=" + who);
                            if (!who.Equals(lstate.holder))
                                throw new IsisException("Locking package exception: RELEASE by " + who + " of " + lockName + (lstate.islocked ? (" held by " + lstate.holder.ToString()) : " but it wasn't locked"));
                            lstate.islocked = false;
                            if (lstate.wantLock.Count() > 0)
                                grantIt = true;
                            break;
                        case CANCEL:
                            if ((IsisSystem.Debug & IsisSystem.LOCKS) != 0)
                                Isis.WriteLine(minfo + "; CANCEL<" + cntr / 10000 + "." + cntr % 1000 + ">: " + lockName + ", who=" + who);
                            if (lstate.wantLock.Contains(who))
                                cancelIt = true;
                            break;
                    }
                }
                if (grantIt)
                    GrantLock(lockName, lstate);
                else if (cancelIt)
                    CancelLockRequest(lockName, who, lstate);
            });

            doRegister(Isis.SGAGGREGATE, (Action<Address, bool, int, int, byte[]>)delegate(Address from, bool fromBelow, int level, int vid, byte[] ba)
            {
                object[] objs = Msg.BArrayToObjects(ba);
                object key = objs[0];
                object value = objs[1];
                int thevid, lcnt = 0;
                do
                {
                    if (lcnt++ > 0)
                        // The issue here is that when a new view is delivered it takes time to reinitialize the aggregation structures and to
                        // unwind existing aggregation waits and meanwhile, an early SGAGGREGATE message could sneak through.  This clumsy 
                        // mechanism will wait as much as 10 secs for that to finish.
                        Thread.Sleep(1000);
                    using (new LockAndElevate(ViewLock))
                        thevid = theView.viewid;
                }
                while (vid > thevid && lcnt < 10);
                if (vid > thevid)
                    throw new IsisException("SAGGREGATE: received a p2p message for view " + vid + " but I seem to be stuck in view " + thevid);
                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("SGAGGREGATE from " + from + ": level=" + level + ", " + (fromBelow ? "DValue" : "RValue") + ", vid=" + vid + " (current vid " + thevid + "), key=" + key + ", value=" + value);
                IAggregateEventHandler callme = null;
                using (new LockAndElevate(AggListLock))
                    if (level > AggList.Length)
                        return;
                    else
                        foreach (IAggregateEventHandler iae in AggList[level])
                            if (iae.GetKeyType().Equals(key.GetType()) && iae.GetValueType().Equals(value.GetType()))
                            {
                                callme = iae;
                                break;
                            }
                if (callme != null)
                    callme.GotSGAggInfo(fromBelow, level, vid, key, value);
                else if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("SGAGGREGATE: WARNING.... couldn't find the callback handler");
            });

            RegisterMakeChkpt((ChkptMaker)delegate(View v)
            {
                if (!LockingInUse)
                    return;
                string[] lnames, wnames = null;
                LockInfo[] linfo = null;
                Address[] wAddr = null;
                using (new LockAndElevate(LocksListLock))
                {
                    lnames = LocksList.Keys.ToArray();
                    if (lnames != null)
                    {
                        linfo = LocksList.Values.ToArray();
                        int wcnt = 0;
                        foreach (LockInfo li in linfo)
                            wcnt += li.wantLock.Count();
                        wnames = new string[wcnt];
                        wAddr = new Address[wcnt];
                        wcnt = 0;
                        foreach (LockInfo li in linfo)
                        {
                            foreach (Address a in li.wantLock)
                            {
                                wnames[wcnt] = li.name;
                                wAddr[wcnt++] = a;
                            }
                        }
                    }
                }
                if ((IsisSystem.Debug & IsisSystem.LOCKS) != 0)
                    Isis.WriteLine("Sending lock state<" + gname + ">:\r\n" + GetLockState());
                if (lnames != null)
                    SendChkpt(lnames, linfo, wnames, wAddr);
            });

            RegisterLoadChkpt((Action<string[], LockInfo[], string[], Address[]>)delegate(string[] keys, LockInfo[] linfo, string[] names, Address[] addrs)
            {
                LockingInUse = true;
                using (new LockAndElevate(LocksListLock))
                {
                    for (int n = 0; n < keys.Length; n++)
                        LocksList[keys[n]] = linfo[n];
                    for (int n = 0; n < names.Length; n++)
                        LocksList[names[n]].wantLock.Add(addrs[n]);
                }
                if ((IsisSystem.Debug & IsisSystem.LOCKS) != 0)
                    Isis.WriteLine("After loading lock state<" + gname + ">:\r\n" + GetLockState());
            });

            // This client-of-a-group to group-representative relaying logic
            doRegister(Isis.RELAYSEND, (Action<Msg>)delegate(Msg m)
            {
                if ((IsisSystem.Debug & IsisSystem.RELAYLOGIC) != 0)
                    Isis.WriteLine("Group owner relaying multicast(" + m + ")");
                doSend(false, false, Isis.RELAYSEND, Isis.my_address, new Msg(m));
            });

            doRegister(Isis.RELAYSEND, (Action<Address, Msg>)delegate(Address leader, Msg m)
            {
                // Runs in all the members
                if ((IsisSystem.Debug & IsisSystem.RELAYLOGIC) != 0)
                    Isis.WriteLine("Large Group owner delivering relayed multicast(" + m + ")");
                List<Msg> newRelayedLGSends = new List<Msg>();
                using (new LockAndElevate(RelayedLGSendsLock))
                {
                    foreach (Msg rm in RelayedLGSends)
                        if (rm != m && rm.vid != -1 && rm.msgid != -1)
                            newRelayedLGSends.Add(rm);
                    RelayedLGSends = newRelayedLGSends;
                }
                if ((IsisSystem.Debug & IsisSystem.DELIVERY) != 0)
                    Isis.WriteLine("<" + gname + ">: UnpackAndDeliver delivering " + m);
                doDeliveryCallbacks(m, "from UnpackAndDeliver (for relayed large-group sends)", Msg.FIFOCAST);
            });

            if ((flags & G_ISLARGE) != 0)
                RegisterViewHandler((ViewHandler)delegate(View v)
                {
                    tokenInfo theToken;
                    using (new LockAndElevate(TokenLock))
                        theToken = this.theToken;
                    if (theToken != null)
                    {
                        if (prevLGOwner != null && theToken.groupOwner.Equals(prevLGOwner))
                            return;
                        prevLGOwner = theToken.groupOwner;
                        List<Msg> LGResendList = new List<Msg>();
                        using (new LockAndElevate(RelayedLGSendsLock))
                            foreach (Msg rm in RelayedLGSends)
                                LGResendList.Add(rm);
                        foreach (Msg m in LGResendList)
                            P2PSend(theToken.groupOwner, Isis.RELAYSEND, m);
                    }
                });

            ViewHandlers += (ViewHandler)LockNewView;

            groupP2PReaderThread = new Thread(delegate()
            {
                try
                {
                    Thread.CurrentThread.Name = "<" + gname + "> incoming p2p delivery thread";
                    isP2PThread = true;
                    doDelivery(incomingP2P, "incomingP2P");
                }
                catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                IsisSystem.ThreadTerminationMagic();
            });
            groupP2PReaderThread.Start();

            groupIPMCReaderThread = new Thread(delegate()
            {
                try
                {
                    Thread.CurrentThread.Name = "<" + gname + "> incoming multicasts delivery thread";
                    doDelivery(incomingSends, "incomingSends");
                }
                catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                IsisSystem.ThreadTerminationMagic();
            });
            groupIPMCReaderThread.Start();
        }

        private void AcceptInitialView(Address sender, View theView, int[] mm, int initialSeqn)
        {
            if (HasFirstView)
                return;
            IsisSystem.IsisRestarting = false;
            if ((flags & G_ISLARGE) != 0)
                theView.NextIncomingMsgID[1] = initialSeqn;
            if (this == Isis.ISISMEMBERS)
                IPMCNewView(gaddr, theView);
            if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                Isis.WriteLine("Received a new single-group INITIALVIEW event in group " + gname + " from " + sender + ", View " + theView + ", with initialSeqn = " + initialSeqn);
            if (Isis.WORKER_MODE)
            {
                TypeSig = TypeSignature(this);
                GroupOpen = true;
            }
            if (theView.viewid == 0)
            {
                if (myCheckpointFile != null && CheckPointFileExists())
                    LoadCheckpointFromFile();
                else if (theInitializer != null && !initializationDone)
                    InitializeGroup(theView);
            }
            if (!theView.joiners.Contains(Isis.my_address))
                theView.joiners = new Address[] { Isis.my_address };
            NewView(theView, "single-view initializer", mm);
            TPGroupsLearnedMM(mm);
        }

        private void opqDrain()
        {
            opqDeliver(Isis.NULLADDRESS, 0, 0, 0, Isis.NULLADDRESS, true);
            using (new LockAndElevate(OrderedSubsetListLock))
                OrderedSubsetPQ = new SortedList<osspq, Msg>();
            using (new LockAndElevate(RecentOpqNodesLock))
                RecentOpqNodes = new List<osspq>();
        }

        private void opqDeliver(Address sentBy, int vid, int msgid, long cts, Address ctwho)
        {
            opqDeliver(sentBy, vid, msgid, cts, ctwho, false);
        }

        private void opqDeliver(Address sentBy, int vid, int msgid, long cts, Address ctwho, bool drainOPQ)
        {
            osspq opq = null;
            Msg m = null;
            myTS = Math.Max(myTS, cts);
            List<Msg> msgs = new List<Msg>();
            List<osspq> keys = new List<osspq>();
            using (new LockAndElevate(OrderedSubsetListLock))
            {
                if (!sentBy.isNull())
                {
                    foreach (var kvp in OrderedSubsetPQ)
                        if (kvp.Value.sender.Equals(sentBy) && kvp.Value.vid == vid && kvp.Value.msgid == msgid)
                        {
                            opq = kvp.Key;
                            m = kvp.Value;
                            break;
                        }
                    if (opq != null)
                    {
                        OrderedSubsetPQ.Remove(opq);
                        opq.commit(cts, ctwho);
                        if (!OrderedSubsetPQ.ContainsKey(opq))
                            OrderedSubsetPQ.Add(opq, m);
                    }
                    else if ((IsisSystem.Debug & IsisSystem.ORDEREDSEND) != 0)
                        Isis.WriteLine("opqDeliver CAN'T FIND (" + sentBy + "::" + vid + ":" + msgid + ", cts=" + cts + "::" + ctwho + ")");
                }
                while (OrderedSubsetPQ.Count() > 0 && (drainOPQ || OrderedSubsetPQ.ElementAt(0).Key.cflag))
                {
                    var first = OrderedSubsetPQ.ElementAt(0);
                    OrderedSubsetPQ.Remove(first.Key);
                    if (first.Key.cflag && first.Value != null)
                    {
                        if ((IsisSystem.Debug & IsisSystem.ORDEREDSEND) != 0)
                            Isis.WriteLine("ORDERED SUBSET CAST: " + (drainOPQ ? "[DRAIN during flush] " : "") + "Delivery of " + first.Value);
                        first.Value.type = Msg.REDELIVERY;
                        msgs.Add(first.Value);
                    }
                    else if ((IsisSystem.Debug & (IsisSystem.ORDEREDSEND | IsisSystem.DISCARDS)) != 0)
                        Isis.WriteLine("ORDERED SUBSET CAST: " + (drainOPQ ? "[DISCARD during flush] " : "") + "Delivery of " + first.Value);
                    if (!drainOPQ)
                        keys.Add(first.Key);
                }
            }
            foreach (Msg msg in msgs)
                incomingP2P.put(msg);
            using (new LockAndElevate(RecentOpqNodesLock))
                foreach (var key in keys)
                    RecentOpqNodes.Add(key);
        }

        private void InitializeGroup(View theView)
        {
            if (initializationDone)
                return;
            initializationDone = true;
            if (theInitializer != null)
                theInitializer();
            if (myCheckpointFile != null && theView.IAmLeader())
                MakeCheckpoint(theView);
        }

        private string OrderToString(Address[] senders, int[] vids, int[] msgids)
        {
            string s = "{ ";
            for (int n = 0; n < senders.Length; n++)
                s += senders[n] + "::" + vids[n] + ":" + msgids[n] + " ";
            return s + "}";
        }

        // Caller must hold the SSLock
        private void deliverSSItems()
        {
            List<Msg> toDeliver = new List<Msg>();
            KeyValuePair<SUTW, Msg> ssItem;
            while ((ssItem = SSList.ElementAtOrDefault(0)).Value != null && ssItem.Key.commitFlag)
            {
                if (!SSList.Remove(ssItem.Key))
                    throw new IsisException("SSList.Remove failed");
                if (durabilityMethod != null)
                    durabilityMethod.SetOrder(ssItem.Value);
                Msg m = ssItem.Value;
                m.type = Msg.REDELIVERY;
                toDeliver.Add(m);
                if ((IsisSystem.Debug & IsisSystem.SAFESEND) != 0)
                    Isis.WriteLine("deliverSSItems: delivering " + m.sender + "::" + m.vid + ":" + m.msgid);
                if (incomingSends.LastPushedVID != 0 && m.vid != incomingSends.LastPushedVID)
                    throw new IsisException("Msg.REDELIVERY/SAFESEND pushing " + m.sender + "::" + m.vid + ":" + m.msgid + " while incomingSends.viewid=" + incomingSends.LastPushedVID);
            }
            incomingSends.putFront(toDeliver);
        }

        internal void CheckRetainedOpqINodes()
        {
            List<osspq> newropq = new List<osspq>();
            using (new LockAndElevate(RecentOpqNodesLock))
            {
                foreach (osspq opq in RecentOpqNodes)
                    if (opq.gtime == 0)
                    {
                        // This worries me; better would be a more explicit garbage collection based on
                        // tracking the completion of pending sends of the ORDEREDSEND commit times and then
                        // disseminating that information using the same background gossip mechanisms that
                        // we use to check rates.  In fact this is probably pretty safe but as a purist,
                        // it bugs me that coded in this manner, the system would potentially violate its
                        // properties if a crash disrupts sending of the ORDEREDSEND commits, and then the
                        // associated flush doesn't start "soon enough".  A further issue is that the list
                        // itself could get long.
                        opq.gtime = Isis.NOW() + 60000;
                        newropq.Add(opq);
                    }
                    else if (Isis.NOW() < opq.gtime)
                        newropq.Add(opq);
                RecentOpqNodes = newropq;
            }
        }

        private void EndStateXfer()
        {
            using (new LockAndElevate(GroupFlagsLock))
            {
                if ((flags & G_NEEDSTATEXFER) != 0)
                    xferWait.Release();
                flags &= ~G_NEEDSTATEXFER;
            }
            ReplayToDo();
        }

        internal static void GroupNoteFailure(Group g, Address which)
        {
            int r = -1;
            using (new LockAndElevate(g.ViewLock))
                if (g.theView != null)
                    r = g.theView.GetRankOf(which);
            if (r != -1)
            {
                bool doCB = false;
                using (new LockAndElevate(Isis.RIPLock))
                    if (Isis.RIPList.Contains(which) == false)
                    {
                        Isis.RIPList.Add(which);
                        doCB = true;
                    }
                if (doCB)
                {
                    ILock.NoteFailed(g.gaddr, which);
                    ReliableSender.P2PSequencer.remoteFailed(which);
                    ReliableSender.AckNoteFailure(which);
                    View.noteFailed(g, which);
                    AwaitReplies.doNoteFailure(which);
                    Group.deFragNoteFailure(which);
                }
            }
        }

        // VTa happens before VTb iff
        //     For all i: VTa[i] <= VTb[i]
        //     For some i: VTa[i] < VTb[i]
        internal bool happensBefore(int[] VTa, int[] VTb)
        {
            string vta = VTtoString(VTa), vtb = VTtoString(VTb);
            bool rv = _happensBefore(VTa, VTb);
            if ((IsisSystem.Debug & IsisSystem.CAUSALDELIVERY) != 0)
                Isis.WriteLine("happensBefore(" + vta + ", " + vtb + ")=" + rv);
            return rv;
        }

        private static string VTtoString(int[] VT)
        {
            string vt = "[ ";
            foreach (int i in VT)
                vt += i.ToString("D3") + " ";
            return vt + "]";
        }
        internal bool _happensBefore(int[] VTa, int[] VTb)
        {
            bool res = false;
            if (VTa.Length != VTb.Length)
                throw new IsisException("happensBefore: VT lengths don't match");
            for (int i = 0; i < VTa.Length; i++)
                if (VTa[i] > VTb[i])
                    return false;
                else if (VTa[i] < VTb[i])
                    res = true;
            return res;
        }

        // Incoming causal send is deliverable if next from this sender
        // and all causally prior messages have been delivered
        internal bool isDeliverable(int senderRank, int[] VT)
        {
            using (new LockAndElevate(ViewLock))
            {
                if (VT.Length != theView.myVT.Length || (senderRank < 0 || senderRank >= theView.myVT.Length))
                    throw new IsisException("causal send: isDeliverable fault");
                // Optimistically assume delivery will be possible
                ++theView.myVT[senderRank];
                for (int i = 0; i < VT.Length; i++)
                    if (VT[i] > theView.myVT[i])
                    {
                        // No luck, back out of optimistic pre-increment
                        --theView.myVT[senderRank];
                        return false;
                    }
            }
            return true;
        }

        internal void PrepareForTermination()
        {
            using (new LockAndElevate(quiesceLock))
            {
                if ((flags & G_TERMINATING) != 0)
                    return;
                flags |= G_TERMINATING;
            }
            AwaitReplies.InterruptReplyWaits(this);
            InterruptAggregationWaits();
            while (quiesceCnt-- > 0)
                quiesceWait.WaitOne();
        }

        private bool warnedHim;

        internal bool IsisCallStart()
        {
            using (new LockAndElevate(quiesceLock))
            {
                if ((flags & G_TERMINATING) != 0)
                {
                    if (!warnedHim)
                        Isis.WriteLine("WARNING: <" + gname + "> was closing/terminating but user-code attempted to issue Isis system calls on it");
                    warnedHim = true;
                    return false;
                }
                ++quiesceCnt;
            }
            return true;
        }

        internal void IsisCallDone()
        {
            using (new LockAndElevate(quiesceLock))
            {
                if ((flags & G_TERMINATING) != 0)
                    quiesceWait.Release(1);
                else
                    --quiesceCnt;
            }
        }

        internal void GroupClose()
        {
            if (isTrackingProxy)
            {
                using (new LockAndElevate(TPGroupsLock))
                    TPGroups.Remove(this.gaddr);
                return;
            }
            if (IsisSystem.IsisActive && (gname.Equals("ISISMEMBERS") || !GroupOpen))
                return;
            using (new LockAndElevate(ReliableSender.PendingSendBufferLock))
            {
                GroupOpen = false;
                WasOpen = true;
            }
            if (durabilityMethod != null)
                durabilityMethod.Shutdown();
            using (new LockAndElevate(IsisGroupsLock))
                IsisGroups.Remove(this.gaddr);
            if (theView != null)
            {
                theView.viewid++;
                theView.leavers = theView.members;
                theView.members = new Address[0];
                theView.joiners = new Address[0];
                // This will cause IPMC delivery thread to deliver a final view and then terminate itself
                if (incomingSends != null)
                {
                    incomingSends.put(theView);
                    incomingSends.put(null);
                }
                if (incomingP2P != null)
                {
                    incomingP2P.put(theView);
                    incomingP2P.put(null);
                }
            }
            if (!gname.Equals("ORACLE"))
                using (new LockAndElevate(Group.GroupRIPLock))
                {
                    if (!Group.GroupRIPList.Contains(gaddr))
                    {
                        Group.GroupRIPList.Add(gaddr);
                        Isis.OnTimer(Isis.ISIS_DEFAULTTIMEOUT * ((flags & G_ISLARGE) != 0 ? 10 : 2), delegate()
                        {
                            using (new LockAndElevate(Group.GroupRIPLock))
                                Group.GroupRIPList.Remove(gaddr);
                        });
                    }
                }
            if ((flags & G_ISLARGE) != 0)
                ReliableSender.lgPendingSendCleanup(this);
            else
                ReliableSender.PendingSendCleanup(this, null);
            xferWait.Release(1000);
            Semaphore s = CPSSema;
            if (s != null)
                s.Release(1000);
            MCMDSocket.UnMap(new Address[] { gaddr });
            if (theView != null && incomingSends != null && groupIPMCReaderThread != Thread.CurrentThread)
                groupIPMCReaderThread.Join();
            if (theView != null && incomingP2P != null && groupP2PReaderThread != Thread.CurrentThread)
                groupP2PReaderThread.Join();
        }

        // Locking package
        internal int LockPackageConfig;             // Overall policy control for the package
        internal const int LOCK_INTERNAL = 0;          // Used purely "internally" within a group
        internal const int LOCK_EPHEMERAL_EXTERN = 1;  // Used with some form of "external" resource but state not recovered (ephemeral) after a total failure
        internal const int LOCK_RECOVER_EXTERN = 2;    // Used with an external resource and must be recovered after group crashes, then restarts

        internal const int LOCK_RELEASE = 0;        // Lock is released if the member holding it exits.
        internal const int LOCK_TRANSFER = 1;       // By default, if a member holding a lock crashes, the lock is retained and ownership transfers to the rank-zero member

        internal const int LOCKIT = 0;              // Request to obtain a lock
        internal const int RELEASEIT = 1;           // Release it to a new holder
        internal const int CANCEL = 2;              // Cancel a pending request

        internal LockBroken DefaultCallback;

        /// <exclude></exclude>
        internal class LockInfo
        {
            internal string name;
            internal bool islocked = false;
            internal int policy;
            internal Address holder;
            internal bool requestPending = false;
            internal LockBroken Notify;
            internal List<Address> wantLock = new List<Address>();
            internal Semaphore wait = new Semaphore(0, int.MaxValue);

            public byte[] toBArray()
            {
                return Msg.toBArray(islocked, policy, holder == null ? Isis.NULLADDRESS : holder);
            }

            public LockInfo(byte[] ba)
            {
                object[] objs = Msg.BArrayToObjects(ba, typeof(bool), typeof(int), typeof(Address));
                int idx = 0;
                islocked = (bool)objs[idx++];
                policy = (int)objs[idx++];
                holder = (Address)objs[idx++];
                if (holder.isNull())
                    holder = null;
            }

            internal LockInfo(string name)
            {
                this.name = name;
            }
        }

        internal Dictionary<string, LockInfo> LocksList = new Dictionary<string, LockInfo>(100);
        internal LockObject LocksListLock = new LockObject("LocksListLock");
        internal delegate void LockDel(int action, string lockName, Address who, int counter);
        internal bool LockingInUse = false;
        internal int lockCurrentViewid;

        internal void LockNewView(View v)
        {
            lockCurrentViewid = v.viewid;
            if (v.leavers.Length == 0 || !LockingInUse || v.members.Length == 0)
                return;
            if ((IsisSystem.Debug & IsisSystem.LOCKS) != 0)
                Isis.WriteLine("LockNewView " + v);
            List<LockInfo> toGrant = new List<LockInfo>();
            using (new LockAndElevate(LocksListLock))
                foreach (KeyValuePair<string, LockInfo> kvp in LocksList)
                {
                    LockInfo li = kvp.Value;
                    if (li.islocked && v.GetRankOf(li.holder) == -1)
                        switch (li.policy)
                        {
                            case LOCK_TRANSFER:
                                LockBroken lb = li.Notify ?? DefaultCallback;
                                if (lb != null)
                                    lb(LOCK_TRANSFER, li.name, li.holder);
                                li.holder = v.members[0];
                                continue;

                            case LOCK_RELEASE:
                                lb = li.Notify ?? DefaultCallback;
                                if (lb != null)
                                    lb(LOCK_RELEASE, li.name, li.holder);
                                li.holder = null;
                                li.islocked = false;
                                if (li.wantLock.Count() > 0)
                                    toGrant.Add(li);
                                continue;
                        }
                }
            if ((IsisSystem.Debug & IsisSystem.LOCKS) != 0)
                Isis.WriteLine("After NEWVIEW state=\r\n" + GetLockState());
            foreach (LockInfo li in toGrant)
            {
                if ((IsisSystem.Debug & IsisSystem.LOCKS) != 0)
                    Isis.WriteLine("After NEWVIEW GrantLock " + li.name);
                GrantLock("Name not known", li);
            }
        }

        /// <summary>
        /// Obtains a lock within a group on a resource, given a string naming that resource
        /// </summary>
        /// <param name="lockName">Name of the lock; any string will do</param>
        /// <returns>True if successful, false for failure</returns>
        /// <remarks>Lock(x) can fail in two ways: by timing out, or (if LOCK_EXTERN_EPHEMERAL was specified) in the event
        /// that the group being used drops below the SafeSendThreshold, which causes SafeSend to fail.</remarks>
        public bool Lock(string lockName)
        {
            return Lock(lockName, int.MaxValue);
        }

        /// <summary>
        /// Obtains a lock; gives up if the timeout expires
        /// </summary>
        /// <param name="lockName">Name of the lock; any string will do</param>
        /// <param name="timeout">Timeout in ms</param>
        /// <returns>True if successful, false for failure</returns>
        /// <remarks>Lock(x) can fail in two ways: by timing out, or (if LOCK_EXTERN_EPHEMERAL was specified) in the event
        /// that the group being used drops below the SafeSendThreshold, which causes SafeSend to fail.</remarks>
        public bool Lock(string lockName, int timeout)
        {
            if (!IsisCallStart())
                return false;
            if ((IsisSystem.Debug & IsisSystem.LOCKS) != 0)
                Isis.WriteLine("Lock[" + gname + "](" + lockName + "; timeout=" + ((timeout == int.MaxValue) ? "*" : timeout.ToString()) + ")\r\n" + GetLockState());
            if ((flags & G_ISLARGE) != 0)
                throw new IsisException("Locking: Not support in large groups");
            LockingInUse = true;
            int tid = -1;
            LockInfo lstate;
            using (new LockAndElevate(LocksListLock))
            {
                if (!LocksList.TryGetValue(lockName, out lstate))
                {
                    lstate = new LockInfo(lockName);
                    LocksList[lockName] = lstate;
                }
                if (!lstate.requestPending)
                    lstate.requestPending = true;
                else
                    throw new IsisException("g.Lock(" + lockName + "): Reentrancy not currently supported");
            }
            bool LockSendFailed = false;
            try
            {
                LockSend(LOCKIT, lockName, Isis.my_address);
            }
            catch (IsisSafeSendException)
            {
                // Occurs in a group where SafeSendThreshold isn't set to ALL and where
                // LOCK_EXTERN_EPHEMERAL was specified if the group size drops below SafeSendThreshold
                LockSendFailed = true;
            }
            if (timeout < int.MaxValue)
                tid = Isis.OnTimerThread(Math.Max(100, timeout), (TimerCallback)delegate()
                {
                    LockSend(CANCEL, lockName, Isis.my_address);
                });
            if (!LockSendFailed)
            {
                // Wait for the lock, but give up if the group starts to Terminate.
                while (lstate.wait.WaitOne(1000) == false)
                    if ((flags & G_TERMINATING) != 0)
                    {
                        IsisCallDone();
                        return false;
                    }
            }
            if ((IsisSystem.Debug & IsisSystem.LOCKS) != 0)
                Isis.WriteLine("After Lock(" + lockName + "): got-lock=" + (lstate.islocked && lstate.holder.isMyAddress()) + ":\r\n" + GetLockState());
            using (new LockAndElevate(LocksListLock))
            {
                if (tid != -1)
                    Isis.TimerCancel(tid);
                lstate.requestPending = false;
                IsisCallDone();
                return lstate.islocked && lstate.holder.isMyAddress();
            }
        }

        /// <summary>
        /// Releases a lock
        /// </summary>
        /// <param name="lockName">Name of the lock to be released</param>
        public void Unlock(string lockName)
        {
            if (!IsisCallStart())
                return;
            if ((flags & G_ISLARGE) != 0)
                throw new IsisException("Locking: Not support in large groups");
            LockingInUse = true;
            LockInfo lstate;
            if ((IsisSystem.Debug & IsisSystem.LOCKS) != 0)
                Isis.WriteLine("UnLock[" + gname + "](" + lockName + ")");
            using (new LockAndElevate(LocksListLock))
                if (!LocksList.TryGetValue(lockName, out lstate) || !lstate.islocked)
                    throw new IsisException("g.Unlock(" + lockName + "), but it wasn't locked in:\r\n" + GetLockState());
            LockSend(RELEASEIT, lockName, Isis.my_address);
            if ((IsisSystem.Debug & IsisSystem.LOCKS) != 0)
                Isis.WriteLine("After Unlock(" + lockName + "):\r\n" + GetLockState());
            IsisCallDone();
        }

        private static int COUNTER;

        internal void LockSend(int command, string lname, Address who)
        {
            COUNTER++;
            switch (LockPackageConfig)
            {
                case LOCK_INTERNAL:
                    OrderedSend(Isis.LOCKREQ, command, lname, who, Isis.my_pid * 10000 + COUNTER);
                    return;
                case LOCK_EPHEMERAL_EXTERN:
                    SafeSend(Isis.LOCKREQ, command, lname, who, Isis.my_pid * 10000 + COUNTER);
                    return;
                case LOCK_RECOVER_EXTERN:
                    SafeSend(Isis.LOCKREQ, command, lname, who, Isis.my_pid * 10000 + COUNTER);
                    return;
            }
        }

        internal void GrantLock(string lockName, LockInfo lstate)
        {
            string toWhom = null;
            Address who;
            using (new LockAndElevate(LocksListLock))
            {
                who = lstate.wantLock.First();
                if (lstate.wantLock.Remove(who) == false)
                    throw new IsisException("GrantLock: unable to remove " + who);
                lstate.islocked = true;
                lstate.holder = who;
                if (who.isMyAddress())
                {
                    toWhom = " GRANTED TO ME!";
                    lstate.wait.Release(1);
                }
                else
                    toWhom = " to " + who;
            }
            if ((IsisSystem.Debug & IsisSystem.LOCKS) != 0)
                Isis.WriteLine("-------LOCK " + lockName + toWhom);
            if (LockPackageConfig == LOCK_RECOVER_EXTERN)
                MakeCheckpoint(theView);
        }

        internal void CancelLockRequest(string LockName, Address who, LockInfo lstate)
        {
            if ((IsisSystem.Debug & IsisSystem.LOCKS) != 0)
                Isis.WriteLine("-------CANCEL " + LockName + " request by " + who);
            using (new LockAndElevate(LocksListLock))
            {
                if (lstate.wantLock.Remove(who) == false)
                    throw new IsisException("CancelLockRequest: unable to remove " + who);
                if (who.isMyAddress())
                    lstate.wait.Release(1);
            }
            if (LockPackageConfig == LOCK_RECOVER_EXTERN)
                MakeCheckpoint(theView);
        }

        /// <summary>
        /// Given the name of a lock, returns the current holder if the lock is locked.  Returns null if unlocked.
        /// </summary>
        /// <param name="lockName"></param>
        /// <returns>Address of the current holder, if any</returns>
        public Address Holder(string lockName)
        {
            if ((flags & G_ISLARGE) != 0)
                throw new IsisException("Locking: Not support in large groups");
            using (new LockAndElevate(LocksListLock))
            {
                LockInfo state;
                if (LocksList.TryGetValue(lockName, out state) && state.islocked)
                    return state.holder;
            }
            return null;
        }

        internal string GetLockState()
        {
            int cnt = 0;
            string s = "";
            using (new LockAndElevate(LocksListLock))
                foreach (KeyValuePair<string, LockInfo> kvp in LocksList)
                {
                    string wlist = "";
                    if (kvp.Value.wantLock != null)
                        foreach (Address a in kvp.Value.wantLock)
                            wlist += a;
                    s += "      USER-DEFINED LOCK: {" + kvp.Key + "::" + (kvp.Value.islocked ? ("Locked by " + kvp.Value.holder.ToString()) : "Not locked") + (wlist.Length > 0 ? (", wanted-by: " + wlist) : ", no waiting requests") + "}\r\n";
                    ++cnt;
                }
            return cnt > 0 ? s : "";
        }

        /// <summary>
        /// Returns the current state of the locking system for this group as a key-value pair array
        /// </summary>
        /// <returns>the state</returns>
        public KeyValuePair<string, Address>[] LockState()
        {
            List<KeyValuePair<string, Address>> state = new List<KeyValuePair<string, Address>>();
            using (new LockAndElevate(LocksListLock))
                foreach (KeyValuePair<string, LockInfo> kvp in LocksList)
                    state.Add(new KeyValuePair<string, Address>(kvp.Key, kvp.Value.islocked ? kvp.Value.holder : Isis.NULLADDRESS));
            return state.ToArray();
        }

        /// <summary>
        /// Specifies the desired handling in the event that a crash occurs while someone is holding the lock
        /// </summary>
        /// <param name="lockName">Name of the lock</param>
        /// <param name="lockPolicy">LOCK_TRANSFER transfers lock to the rank0 member, LOCK_RELEASE releases the lock </param>
        /// <remarks>It doesn't make sense to use the LOCK_RELEASE policy if the service as a whole is running int the slow,
        /// very conservative, LOCK_RECOVER_EXTERN mode. </remarks>
        public void SetLockPolicy(string lockName, int lockPolicy)
        {
            SetLockPolicy(lockName, lockPolicy, null);
        }
        /// <summary>
        /// Specifies the desired handling in the event that a crash occurs while someone is holding the lock
        /// </summary>
        /// <param name="lockName">Lock name</param>
        /// <param name="lockPolicy">Desired policy</param>
        /// <param name="del">Method to notify in the event of an event</param>
        public void SetLockPolicy(string lockName, int lockPolicy, LockBroken del)
        {
            if ((flags & G_ISLARGE) != 0)
                throw new IsisException("Locking: Not support in large groups");
            LockingInUse = true;
            if (lockPolicy == LOCK_RELEASE && LockPackageConfig == LOCK_RECOVER_EXTERN)
                Isis.WriteLine("WARNING: Lock policy inconsistency: LOCK_RELEASE for lock " + lockName + " in LOCK_RECOVER_EXTERN mode!");
            using (new LockAndElevate(LocksListLock))
            {
                LockInfo lstate;
                if (!LocksList.TryGetValue(lockName, out lstate))
                {
                    lstate = new LockInfo(lockName);
                    LocksList[lockName] = lstate;
                }
                lstate.policy = lockPolicy;
                lstate.Notify = del;
            }
        }

        /// <summary>
        /// Parameters controlling the overall package
        /// </summary>
        /// <param name="config">LOCK_INTERNAL, LOCK_EPHEMERAL_EXTERN, LOCK_RECOVER_EXTERN</param>
        /// <remarks>The SetLockPolicies control API allows the lock package user to indicate to Isis2 how the locking system is being
        /// used.  With configuration parameter LOCK_INTERN, locking is understood to run on a "soft state" resource with purely in-memory
        /// state.  This is fastest, but is safe only if you have good reason to know that your group won't need to tolerate failures
        /// in which all members fail.  We recommend it for groups that have some "internal" synchronization need, but there may be
        /// cloud settings in which this is a safe choice based in information you might have about the likelihood of a total failure.
        /// 
        /// With the LOCK_EPHEMERAL_EXTERN policy, the locking package is configured to deal with an external resource but still forgets
        /// the lock state in the event of a total failure.  This is slower than LOCK_INTERNAL but not dramatically so, and has the advantage
        /// that it can safely be used if the goal is to lock access to, say, a printer.  Locks are released if the service crashes but if you
        /// know that the external resource won't be accessed during those periods, you may be safe with this choice.
        /// 
        /// The slowest but safest option is LOCK_REOVER_EXTERN.  Here we are pretty obssessive about logging state and can restore the service
        /// after any possible failure with the same lock state it was in prior to the failure.  Costs are high, and this makes sense only with the
        /// LOCK_TRANSFER option for handling of failures, since ALL the old members will have crashed.  If you would use LOCK_RELEASE as a policy,
        /// you definitely don't need LOCK_RECOVER_EXTERN as a configuration state. </remarks>
        public void SetLockPolicies(int config)
        {
            SetLockPolicies(config, null);
        }

        /// <summary>
        /// Parameters controlling the overall packageParameters controlling the overall package
        /// </summary>
        /// <param name="config">LOCK_INTERNAL, LOCK_EPHEMERAL_EXTERN, LOCK_RECOVER_EXTERN</param>
        /// <param name="del">Default callback for lock transfer or broken events</param>
        /// <remarks>The SetLockPolicies control API allows the lock package user to indicate to Isis2 how the locking system is being
        /// used.  With configuration parameter LOCK_INTERN, locking is understood to run on a "soft state" resource with purely in-memory
        /// state.  This is fastest, but is safe only if you have good reason to know that your group won't need to tolerate failures
        /// in which all members fail.  We recommend it for groups that have some "internal" synchronization need, but there may be
        /// cloud settings in which this is a safe choice based in information you might have about the likelihood of a total failure.
        /// 
        /// With the LOCK_EPHEMERAL_EXTERN policy, the locking package is configured to deal with an external resource but still forgets
        /// the lock state in the event of a total failure.  This is slower than LOCK_INTERNAL but not dramatically so, and has the advantage
        /// that it can safely be used if the goal is to lock access to, say, a printer.  Locks are released if the service crashes but if you
        /// know that the external resource won't be accessed during those periods, you may be safe with this choice.
        /// 
        /// The slowest but safest option is LOCK_REOVER_EXTERN.  Here we are pretty obssessive about logging state and can restore the service
        /// after any possible failure with the same lock state it was in prior to the failure.  Costs are high, and this makes sense only with the
        /// LOCK_TRANSFER option for handling of failures, since ALL the old members will have crashed.  If you would use LOCK_RELEASE as a policy,
        /// you definitely don't need LOCK_RECOVER_EXTERN as a configuration state. </remarks>
        public void SetLockPolicies(int config, LockBroken del)
        {
            if ((flags & G_ISLARGE) != 0)
                throw new IsisException("Locking: Not support in large groups");
            LockingInUse = true;
            LockPackageConfig = config;
            DefaultCallback = del;
            switch (config)
            {
                case LOCK_INTERNAL:
                    // This is the default
                    return;
                case LOCK_EPHEMERAL_EXTERN:
                    // Forces use of SafeSend
                    return;
                case LOCK_RECOVER_EXTERN:
                    // Uses SafeSend+Logging
                    if (safeSendDurabilityMethod == null)
                        throw new IsisException("Recoverable mode for lock package requires SafeSend DiskDurabilityMode");
                    return;
                default:
                    throw new IsisException("Unrecognized lock policy configuration value: " + config);
            }
        }

        /// <exclude></exclude>
        [AutoMarshalled]
        public class DHTItem
        {
            /// <exclude></exclude>
            public object value;
            /// <exclude></exclude>
            public Address createdBy;
            /// <exclude></exclude>
            public long createTime;

            /// <exclude></exclude>
            public DHTItem()
            {
            }

            internal DHTItem(object val)
            {
                createdBy = Isis.my_address;
                createTime = Isis.CurTOD();
                value = val;
            }

            internal DHTItem(Address who, long when, object val)
            {
                createdBy = who;
                createTime = when;
                value = val;
            }
        }

        internal Dictionary<object, DHTItem> DHTContents = new Dictionary<object, DHTItem>(1000);

        /// <summary>
        /// Returns a clone containing only those (key,value) tuples that have the specified key and value types, or the full set if
        /// the type is specified as "object".   The result is a copy: changes to it won't be reflected into 
        /// the "original". which is maintained internally by Isis and not directly accessible to the user.
        /// <typeparam name="KT">Key type</typeparam>
        /// <typeparam name="VT">Value type</typeparam>
        /// </summary>
        public IEnumerable<KeyValuePair<KT, VT>> DHT<KT, VT>()
        {
            List<KeyValuePair<KT, VT>> rval = new List<KeyValuePair<KT, VT>>();
            if (!IsisCallStart())
                return rval;
            using (new LockAndElevate(DHTDictLock))
                foreach (var kvp in DHTContents)
                    if (kvp.Key.GetType().Equals(typeof(KT)) && kvp.Value.value.GetType().Equals(typeof(VT)))
                        rval.Add(new KeyValuePair<KT, VT>((KT)kvp.Key, (VT)kvp.Value.value));
            IsisCallDone();
            return rval;
        }
        /// <summary>
        /// Returns a clone containing of the Isis DHT
        /// </summary>
        public IEnumerable<KeyValuePair<object, object>> DHT()
        {
            List<KeyValuePair<object, object>> rval = new List<KeyValuePair<object, object>>();
            if (!IsisCallStart())
                return rval;
            using (new LockAndElevate(DHTDictLock))
                foreach (var kvp in DHTContents)
                    rval.Add(new KeyValuePair<object, object>(kvp.Key, kvp.Value.value));
            IsisCallDone();
            return rval;
        }

        /// <summary>
        /// Returns an integer (small non-negative numbers 0... nmembers/replication factor) indicating which DHT partition this member belongs to
        /// </summary>
        /// <returns></returns>
        public int DHTGetPartition()
        {
            return GetAffinityGroup(GetMyRank());
        }

        /// <summary>
        /// Returns an integer (small non-negative numbers 0... nmembers/replication factor) indicating my rank within my partition
        /// </summary>
        /// <returns></returns>
        public int DHTGetPartitionRank()
        {
            View theView;
            using (new LockAndElevate(ViewLock))
                theView = this.theView;
            int rank = 0, myAg = GetAffinityGroup(GetMyRank());
            if (myAg >= 0)
                for (int i = 0; i < theView.members.Length; i++)
                    if (theView.members[i].isMyAddress())
                        return rank;
                    else if (GetAffinityGroup(i) == myAg)
                        ++rank;
            return -1;
        }


        internal LockObject DHTDictLock = new LockObject("DHTLock");
        internal int myDHTBinSize, myDHTMinSize, myDHTnShards;
        internal Func<object, object, object, object> myResolver;
        internal View curDHTView;
        internal LockObject DHTViewLock = new LockObject("DHTViewLock");
        internal bool myDHTInDebugMode = false;
        internal bool[,] DHTHMaps;
        internal bool[] DHTAgNonEmpty;
        internal int myTargetGroupSize;
        internal const int DHTRedundancyFactor = 2;               // How many existing members push DHT content to you when membership shuffles due to churn
        internal long myDHTItemTimeout;

        // Caller holds the lock, so no need to acquire it here
        internal void DHTWriter(object key, DHTItem value)
        {
            bool collision;
            DHTItem dhi = null;
            collision = DHTContents.ContainsKey(key);
            if (collision)
                dhi = DHTContents[key];
            if ((IsisSystem.Debug & IsisSystem.DHTS) != 0)
                Isis.WriteLine("DHT_PUT: kvp=<" + key + "::" + value.value + ">" + (collision ? " ** COLLISION **" : ""));
            if (collision)
            {
                bool resolved = false;
                object newValue = null;
                // If the resolver was specified, let it decide what to do
                if (theResolvers.hList.Count() > 0)
                {
                    object[] obs = new object[] { key, dhi.value, value.value };
                    foreach (var del in theResolvers.hList)
                    {
                        if (DHTTypeMatch(obs, del) && del.cbProc.nParams == 3)
                        {
#if !(__MonoCS__ || ___ANDROID__)
                            if (del.cbProc.cb != null)
                                newValue = ((dynamic)del.cbProc.cb).Invoke((dynamic)obs[0], (dynamic)obs[1], (dynamic)obs[2]);
                            else
                            {
                                MethodInfo mi = del.cbProc.hisCb.GetType().GetMethod("Invoke");
                                newValue = mi.Invoke(del.cbProc.hisCb, obs);
                            }
#else
                            newValue = del.cbProc.hisCb.DynamicInvoke(obs);
#endif
                            resolved = true;
                            break;
                        }
                    }
                }
                if (!resolved && myResolver != null)
                {
                    newValue = myResolver(key, dhi.value, value.value);
                    resolved = true;
                }
                if (resolved)
                {
                    DHTContents.Remove(key);
                    if (!usesDHTDefaults)
                        DHTExtWriter(key, null);
                    if (newValue == null)
                        return;
                    if (usesDHTDefaults)
                        DHTContents.Add(key, new DHTItem(value.createdBy, value.createTime, newValue));
                    else
                        DHTContents.Add(key, new DHTItem(value.createdBy, value.createTime, DHTExtWriter(key, newValue)));
                    return;
                }
                // If two puts are done on the same key, keep the Put with the larger create time.  This is a wall-clock time of day
                // In the whole of Isis2, this is the only line of code sensitive to the quality of clock synchronization....
                if (dhi.createTime >= value.createTime)
                    return;
                if (DHTContents.Remove(key) == false)
                    Isis.WriteLine("WARNING: unexpected failure to remove " + key);
                if (!usesDHTDefaults)
                    DHTExtWriter(key, null);
            }
            if (usesDHTDefaults)
                DHTContents.Add(key, value);
            else
                DHTContents.Add(key, new DHTItem(value.createdBy, value.createTime, DHTExtWriter(key, value.value)));
        }

        internal object DHTReader(object key)
        {
            DHTItem dhi = null;
            using (new LockAndElevate(DHTDictLock))
                DHTContents.TryGetValue(key, out dhi);
            if ((IsisSystem.Debug & IsisSystem.DHTS) != 0)
                Isis.WriteLine("DHT_GET: key=<" + key + ">" + ((dhi != null ? " found (value=" + dhi.value + ")" : " not found")));
            if (dhi != null)
            {
                if (usesDHTDefaults)
                    return dhi.value;
                return DHTExtReader(key, (string)dhi.value);
            }
            else
                return null;
        }

        internal bool usesDHTDefaults = true;

        /// <summary>
        /// By specifying a DHTPutMethod, the user can provide a method that stores the actual value objects
        /// For example, they could be stored in external files.  The PutMethod returns a locator string
        /// </summary>
        /// <param name="Key">The key of the object to be stored.  If reused, the object is being overwritten</param>
        /// <param name="Value">The value of the object to be stored.  Null if the object is being deleted</param>
        /// <returns></returns>
        public delegate string DHTPutMethod(object Key, object Value);
        /// <summary>
        /// Called when fetching a value for an externally stored object
        /// </summary>
        /// <param name="Key">Key of the externally stored object</param>
        /// <param name="locator">string that was provided by the DHTPutMethod when it was last saved</param>
        /// <returns></returns>
        public delegate object DHTGetMethod(object Key, string locator);

        internal DHTPutMethod DHTExtWriter;
        internal DHTGetMethod DHTExtReader;

        /// <summary>
        /// Allows a developer to create a DHT that stores its data in a non-standard way.  The default methods keep the DHT data in memory
        /// This API is mostly intended to support DHTs in which the actual values are stored in the local file system by the group members
        /// </summary>
        /// <param name="writerMethod"></param>
        /// <param name="readerMethod"></param>
        public void SetDHTPersistenceMethods(DHTPutMethod writerMethod, DHTGetMethod readerMethod)
        {
            usesDHTDefaults = false;
            DHTExtWriter = writerMethod;
            DHTExtReader = readerMethod;
        }

        /// <summary>
        /// Puts the current group into DHT mode, but without specifying a target size and without setting aside extra members. 
        /// </summary>
        /// <param name="ReplicationFactor">Requested data replication factor</param>
        /// <param name="ExpectedGroupSize">Your estimate of the typical size of this group (N); a multiple of the replication factor</param>
        /// <param name="MinimumGroupSize">The smallest group size at which the DHT will accept DHTPut/DHTGet commands</param>
        /// <remarks>If the replication factor is too small, you run the risk that our random hashing scheme could leave some affinity
        /// group with too few, or too many members.  Too many is not a problem, but if an affinity group has too few members you can see
        /// failures that cause data to be completely lost (e.g. if the only replica fails).  Thus when you ask for a replication factor of,
        /// say, 3 this is a <it>target</it>, not the minimum that might be used.  We haven't had problems with factors of 5 or more in groups of 25 or more members.
        /// 
        /// In the event of membership changes, the group will automatically recompute the DHT hash partitioning function and ship (key,value) pairs around
        /// to ensure that each member has the correct items.  This can be a substantial overhead in heavily populated DHTs with large numbers of members.
        /// In such cases, use the full DHTEnable API and indicate to the system a size you are targetting for the active DHT, and a number of "spare"
        /// members that the system can leave out of the DHT (for example, you might target 1000 members, and allow 5 spares; if the group were to have
        /// 1005 members (or more), N-5 will run the DHT, while the remaining 5 play a backup role.
        /// 
        /// With spares, if a member fails, one of the spares will be dropped in to replace it.  This limits the amount of reinitialization and
        /// "churning" of (key,value) tuples: that spare needs to be initialized, but other members retain their roles and no large-scale churn occurs.
        /// Then you can launch extra spares if the number starts to drop towards your specified target.  
        /// 
        /// When the group has fewer than the ExpectedGroupSize members, the spare mechanism is disabled.  So, continuing our example above, with 950 members, no spares
        /// would be available and the DHT would indeed churn if a failure or join changed the DHT hash-based partitioning of the members.
        /// 
        /// To put the DHT into debug mode for developing your application, you can use ReplicationFactor, ExpectedGroupSize and MinimumGroupSize all set to 1,
        /// and then test with just 1 or 2 members in the group,
        /// but remember to use reasonable values later when your group will have more than 1 member.</remarks>
        public void DHTEnable(int ReplicationFactor, int ExpectedGroupSize, int MinimumGroupSize)
        {
            DHTEnable(ReplicationFactor, ExpectedGroupSize, MinimumGroupSize, int.MaxValue);
        }

        /// <summary>
        /// Puts the current group into DHT mode, but without specifying a target size and without setting aside extra members. 
        /// </summary>
        /// <param name="ReplicationFactor">Requested data replication factor</param>
        /// <param name="ExpectedGroupSize">Your estimate of the typical size of this group (N); a multiple of the replication factor</param>
        /// <param name="MinimumGroupSize">The smallest group size at which the DHT will accept DHTPut/DHTGet commands</param>
        /// <param name="timeout">Timeout, after which the (Key,Value) pair is automatically discarded.</param>
        /// <remarks>If the replication factor is too small, you run the risk that our random hashing scheme could leave some affinity
        /// group with too few, or too many members.  Too many is not a problem, but if an affinity group has too few members you can see
        /// failures that cause data to be completely lost (e.g. if the only replica fails).  Thus when you ask for a replication factor of,
        /// say, 3 this is a <it>target</it>, not the minimum that might be used.  We haven't had problems with factors of 5 or more in groups of 25 or more members.
        /// 
        /// In the event of membership changes, the group will automatically recompute the DHT hash partitioning function and ship (key,value) pairs around
        /// to ensure that each member has the correct items.  This can be a substantial overhead in heavily populated DHTs with large numbers of members.
        /// In such cases, use the full DHTEnable API and indicate to the system a size you are targetting for the active DHT, and a number of "spare"
        /// members that the system can leave out of the DHT (for example, you might target 1000 members, and allow 5 spares; if the group were to have
        /// 1005 members (or more), N-5 will run the DHT, while the remaining 5 play a backup role.
        /// 
        /// With spares, if a member fails, one of the spares will be dropped in to replace it.  This limits the amount of reinitialization and
        /// "churning" of (key,value) tuples: that spare needs to be initialized, but other members retain their roles and no large-scale churn occurs.
        /// Then you can launch extra spares if the number starts to drop towards your specified target.  
        /// 
        /// When the group has fewer than the ExpectedGroupSize members, the spare mechanism is disabled.  So, continuing our example above, with 950 members, no spares
        /// would be available and the DHT would indeed churn if a failure or join changed the DHT hash-based partitioning of the members.
        /// 
        /// To put the DHT into debug mode for developing your application, you can use ReplicationFactor, ExpectedGroupSize and MinimumGroupSize all set to 1,
        /// and then test with just 1 or 2 members in the group,
        /// but remember to use reasonable values later when your group will have more than 1 member.</remarks>
        public void DHTEnable(int ReplicationFactor, int ExpectedGroupSize, int MinimumGroupSize, int timeout)
        {
            if (!IsisCallStart())
                return;
            if (timeout > 0 && timeout < int.MaxValue)
            {
                myDHTItemTimeout = Math.Max(timeout, 2500);
                Isis.OnTimer(myDHTItemTimeout, (TimerCallback)DHTCleanup);
            }
            if (myDHTBinSize != 0)
                throw new IsisDHTException("Can't call DHTEnable more than once for the same group");
            if (ExpectedGroupSize % ReplicationFactor != 0)
                throw new IsisDHTException("Target Isis DHT group size isn't a multiple of the replication factor");
            myDHTMinSize = MinimumGroupSize;
            myDHTBinSize = ReplicationFactor;
            myDHTnShards = ExpectedGroupSize / ReplicationFactor;
            lastVersionId = new int[myDHTnShards];
            retainedKVPS = new List<object[]>[myDHTnShards];
            for (int n = 0; n < myDHTnShards; n++)
                retainedKVPS[n] = new List<object[]>();
            if (MinimumGroupSize < myDHTnShards)
                throw new IsisDHTException("DHTEnable: the minimum size must be at least ExpectedGroupSize/ReplicationFactor");
            myTargetGroupSize = ExpectedGroupSize;
            if (ReplicationFactor == 1 && ExpectedGroupSize == 1 && MinimumGroupSize == 1)
                myDHTInDebugMode = true;
            else if (ReplicationFactor < 2 || myDHTBinSize < 2)
                throw new IsisDHTException("DHTEnable: Replication factor must be >= 2 and (ExpectedGroupSize / ReplicationFactor) must be > 1");
            doRegister(Isis.IM_DHT_PUT, (Action<object>)delegate(object kvps)
            {
                doDHTPut(kvps, 0, -1, -1);
            });
            doRegister(Isis.IM_DHT_PUT, (Action<object, int, int, int>)delegate(object kvps, int sendingShard, int versionId, int viewId)
            {
                doDHTPut(kvps, sendingShard, versionId, viewId);
            });
            doRegister(Isis.IM_DHT_PUT, (Action<int, int, int>)delegate(int sendingShard, int versionId, int viewId)
            {
                doDHTPut(null, sendingShard, versionId, viewId);
            });
            doRegister(Isis.IM_DHT_GET, (Action<object>)delegate(object keys)
            {
                int myAg = GetAffinityGroup(Isis.my_address);
                List<KeyValuePair<object, object>> result = new List<KeyValuePair<object, object>>();
                foreach (var key in (IEnumerable)keys)
                {
                    int khash = DHTKeyHash(key);
                    // Filter if not for my affinity group; shouldn't happen at all
                    if (GetAffinityGroup(khash) == myAg && theView.GetMyRank() < theView.members.Length)
                        using (new LockAndElevate(DHTDictLock))
                        {
                            object value;
                            if ((value = DHTReader(key)) != null)
                                result.Add(new KeyValuePair<object, object>(key, value));
                        }
                    else if ((IsisSystem.Debug & IsisSystem.DHTS) != 0)
                        Isis.WriteLine("DHT_GET hander: filtering and ignoring a get to a different affinity group");

                }
                // This is necessary because of a bug in the Message library: it seems to have a problem with zero-length
                // lists of KeyValuePair<kt,vt> objects and encodes them as zero-length lists of type object..
                if ((IsisSystem.Debug & IsisSystem.DHTS) != 0)
                {
                    string ks = " ";
                    foreach (var key in (IEnumerable)keys)
                        ks += key + "[Ag=" + GetAffinityGroup(key.GetHashCode()) + "] ";
                    Isis.WriteLine("DHT_GET hander[Ag=" + myAg + "]: I was asked about keys " + ks + " and found " + result.Count() + " matches");
                }
                if (result.Count() > 0)
                    doReply(result);
                else
                    doNullReply();
            });
            // Uses a special hook: *= will put the handler at the front of the queue, ensuring that the DHT mapping is
            // recomputed before anyone might try and use it.
            ViewHandlers *= (ViewHandler)delegate(View v)
            {
                if (v.members.Length == 0)
                    return;
                using (new LockAndElevate(DHTViewLock))
                    curDHTView = v;
                int nActive = v.members.Length;
                DHTHMaps = new bool[myDHTnShards, log2(Math.Max(nActive, myDHTnShards * myDHTBinSize)) + 1];
                DHTAgNonEmpty = new bool[myDHTnShards];
                for (int n = 0; n < nActive; n++)
                {
                    int pseudoGrp = GetAffinityGroup(v.members[n]);
                    if (pseudoGrp >= 0)
                    {
                        DHTHMaps[pseudoGrp, log2(n + 1)] = true;
                        DHTAgNonEmpty[pseudoGrp] = true;
                    }
                }
                replaylastSTKVPS(v.viewid);
            };
            // This is special too: we get a chance to shift DHT key-value items around during the PROPOSED step as new views are computed
            FlushHandlers += (ViewHandler)DHTShuffle;

            // Goals here are to (1) Spread the work around, and (2) Make sure each joining member is initialized by a sender
            // belonging to the appropriate affinity group. This code is a bit awkward because it only can be used when the
            // DHT is sufficiently large, and because a DHT with a number of members that isn't an exact multiple of 
            // myDHTnBins might have a few "extra" members at the end.  This is why when rankOfCheckptSender wraps we
            // always want to set it back to hisAg, not to rankOfCheckptSender%v.members.Length!
            RegisterChkptChoser((ChkptChoser)delegate(View v, Address who)
            {
                View prevView;
                using (new LockAndElevate(DHTViewLock))
                    prevView = curDHTView;
                if (v.joiners.Contains(Isis.my_address) || v.members.Length - v.joiners.Length < myDHTMinSize)
                    return false;
                bool fnd = false;
                int hisAg = GetAffinityGroup(v, who);
                int rankOfCheckptSender = (who.GetHashCode() % myDHTnShards) * myDHTnShards + hisAg, off;
                for (off = 0; off < prevView.members.Length / myDHTnShards; off++)
                {
                    if (rankOfCheckptSender >= prevView.members.Length)
                        rankOfCheckptSender = hisAg;
                    if (!v.leavers.Contains(prevView.members[rankOfCheckptSender]))
                    {
                        if (GetAffinityGroup(rankOfCheckptSender) != hisAg)
                            throw new IsisException("Bug in new DHT checkpoint choser (1)");
                        if (fnd = !v.joiners.Contains(prevView.members[rankOfCheckptSender]))
                            break;
                    }
                    rankOfCheckptSender += myDHTnShards;
                }
                if (!fnd)
                    throw new IsisException("Bug in new DHT checkpoint choser (2) with myRank=" + prevView.GetMyRank() +
                        " (Ag=" + GetAffinityGroup(prevView.GetMyRank()) + "), who=" + who + " (Ag=" + hisAg + "), view=" + v);
                return prevView.GetMyRank() == rankOfCheckptSender;
            });
            RegisterMakeChkpt((ChkptMaker)delegate(View v)
            {
                using (new LockAndElevate(DHTDictLock))
                    if (DHTContents.Count() > 0)
                    {
                        if (usesDHTDefaults)
                            // In-memory case; just send the whole thing in one big message
                            SendChkpt(Msg.toBArray(DHTContents.Keys.ToArray()), Msg.toBArray(DHTContents.Values.ToArray()));
                        else
                        {
                            // External user-defined storage.  Here we do the transfer one object at a time because the files
                            // in external storage could be quite large (a common reason for storing them externally)
                            foreach (var kvp in DHTContents)
                                SendChkpt(Msg.toBArray(new object[] { kvp.Key }), Msg.toBArray(new DHTItem[] { new DHTItem(kvp.Value.createdBy, kvp.Value.createTime, DHTExtReader(kvp.Key, (string)kvp.Value.value)) }));
                        }
                    }
                EndOfChkpt();
            });
            RegisterLoadChkpt((Action<byte[], byte[]>)delegate(byte[] kba, byte[] vba)
            {
                object[] keys = Msg.BArrayToObjects(kba);
                object[] values = Msg.BArrayToObjects(vba);
                using (new LockAndElevate(DHTDictLock))
                    for (int i = 0; i < keys.Length; i++)
                        if (usesDHTDefaults)
                        {
                            if (!DHTContents.ContainsKey(keys[i]))
                                DHTContents.Add(keys[i], (DHTItem)values[i]);
                        }
                        else
                            DHTWriter(keys[i], (DHTItem)values[i]);
            });
            IsisCallDone();
        }

        private int[] lastVersionId;
        private int myDHTLastViewId = -1;
        private int myDHTGetHashCodeVID;
        private LockObject myDHTGetHashCodeVIDLock = new LockObject("myDHTGetHashCodeVIDLock");
        private List<object[]>[] retainedKVPS;
        private LockObject lastSTKVPSLock = new LockObject("lastSTKVPSLock");
        private List<object[]> lastSTKVPS = new List<object[]>();
        private Semaphore waitForPutSema = new Semaphore(0, int.MaxValue);

        private void doDHTPut(object kvps, int sendingShard, int versionId, int viewId)
        {
            int myAg = GetAffinityGroup(Isis.my_address);
            using (new LockAndElevate(myDHTGetHashCodeVIDLock))
            {
                if (viewId > myDHTLastViewId)
                {
                    using (new LockAndElevate(lastSTKVPSLock))
                        lastSTKVPS.Add(new object[] { kvps, sendingShard, versionId, viewId });
                    return;
                }
                if (versionId > 0)
                {
                    // This branch detects and ignores DHTGetHashCodeChanged-related duplicates.  Those never
                    // come from sendingShard == my own shard Id, so that one entry in lastVersionId is untouched
                    if (sendingShard == myAg)
                        throw new IsisException("doDHTPut: sendingShard came through as " + sendingShard + ", which is my own shard!");
                    if (versionId <= lastVersionId[sendingShard])
                    {
                        if ((IsisSystem.Debug & IsisSystem.DHTS) != 0)
                            Isis.WriteLine("DHT_PUT hander: filtering and ignoring a duplicated shuffle message");
                        return;
                    }
                    if (versionId > myDHTGetHashCodeVID)
                    {
                        // Arrived early...
                        if ((IsisSystem.Debug & IsisSystem.DHTS) != 0)
                            Isis.WriteLine("DHT_PUT hander: retaining an early shuffle message");
                        retainedKVPS[sendingShard].Add(new object[] { kvps, sendingShard, versionId, viewId });
                        return;
                    }
                    lastVersionId[sendingShard] = versionId;
                }
                else if (versionId == 0)
                {
                    // This branch uses the lastVersionId[myAg] to hold the viewid, and in that way senses dups
                    if (viewId <= lastVersionId[myAg])
                    {
                        if ((IsisSystem.Debug & IsisSystem.DHTS) != 0)
                            Isis.WriteLine("DHT_PUT hander: filtering and ignoring a duplicated state transfer message");
                        return;
                    }
                    if (viewId != lastVersionId[myAg] + 1 && lastVersionId[myAg] != -1)
                        throw new IsisException("in doDHTPut: viewId of " + viewId + " was unexpected: lastVersionId[myAg=" + myAg + "] was " + lastVersionId[myAg]);
                    lastVersionId[myAg] = viewId;
                }
            }
            if (kvps != null)
                foreach (var okvp in (IEnumerable)kvps)
                {
                    object key = okvp.GetType().GetMethod("get_Key").Invoke(okvp, new object[0]);
                    DHTItem value = (DHTItem)okvp.GetType().GetMethod("get_Value").Invoke(okvp, new DHTItem[0]);
                    int khash = DHTKeyHash(key);
                    // Filter if not for my affinity group; won't occur in TCP_ONLY mode
                    View theView;
                    using (new LockAndElevate(ViewLock))
                        theView = this.theView;
                    if (GetAffinityGroup(khash) == GetAffinityGroup(Isis.my_address) && theView.GetMyRank() < theView.members.Length)
                        using (new LockAndElevate(DHTDictLock))
                            DHTWriter(key, value);
                    else if ((IsisSystem.Debug & IsisSystem.DHTS) != 0)
                        Isis.WriteLine("DHT_PUT hander: filtering and ignoring a put to a different affinity group");
                }
            if (versionId > 0)
                waitForPutSema.Release(1);
        }

        private void replaylastSTKVPS(int vid)
        {
            if (vid <= myDHTLastViewId)
                return;
            myDHTLastViewId = vid;
            List<object[]> old;
            using (new LockAndElevate(lastSTKVPSLock))
            {
                old = lastSTKVPS;
                lastSTKVPS = new List<object[]>();
            }
            foreach (var oo in old)
            {
                int idx = 0;
                object kvps = oo[idx++];
                int sendingshard = (int)(oo[idx++]);
                int versionid = (int)(oo[idx++]);
                int viewid = (int)(oo[idx++]);
                if (viewid > vid)
                    using (new LockAndElevate(lastSTKVPSLock))
                        lastSTKVPS.Add(oo);
                else if (viewid == vid)
                    doDHTPut(kvps, sendingshard, versionid, viewid);
            }
        }

        /// <summary>
        /// Called when the application has modified the GetHashCode method used to map keys to shards
        /// </summary>
        /// <param name="versionId">A positive GetHashCode() "version number" that needs to be identical in all the callers.</param>
        /// <remarks>The GetHashCode method needs to change through a series of consistent totally ordered actions, and
        /// for each change, the same version number should be used by all members.  The number must also increase each time
        /// DHTGetHashCodeChanged is called in any given view (for example, you can just use a counter initialized to 1).
        /// </remarks>
        public void DHTGetHashCodeChanged(int versionId)
        {
            View theView;
            using (new LockAndElevate(ViewLock))
                theView = this.theView;
            if (versionId <= 0 || versionId <= lastVersionId[GetAffinityGroup(theView.GetMyRank())])
                throw new IsisDHTException("DHTGetHashCodeChanged: version number must be > 0, and must increase each time this method is called in a given view");
            using (new LockAndElevate(myDHTGetHashCodeVIDLock))
                myDHTGetHashCodeVID = versionId;
            DHTShuffle(theView, versionId, true);
            for (int i = 0; i < myDHTnShards; i++)
            {
                List<object[]> kvpsl;
                using (new LockAndElevate(myDHTGetHashCodeVIDLock))
                {
                    kvpsl = retainedKVPS[i];
                    retainedKVPS[i] = new List<object[]>();
                }
                foreach (var oo in kvpsl)
                {
                    int idx = 0;
                    object kvps = oo[idx++];
                    int sendingshard = (int)(oo[idx++]);
                    int versionid = (int)(oo[idx++]);
                    int viewid = (int)(oo[idx++]);
                    doDHTPut(kvps, sendingshard, versionid, viewid);
                }
            }
            for (int cnt = 1; cnt < myDHTnShards; cnt++)
                waitForPutSema.WaitOne();
        }

        private void DHTShuffle(View nextView)
        {
            DHTShuffle(nextView, 0, false);
        }

        /*
         * Shuffles DHT items that are on the wrong node after a view change or a user change to the GetHashCode() method for keys
         */
        private void DHTShuffle(View nextView, int versionId, bool shuffleMode)
        {
            if (nextView.members.Length == 0)
                return;
            View theView;
            using (new LockAndElevate(ViewLock))
                theView = this.theView;
            if (theView == null)
                return;
            int myOldRank = theView.GetMyRank(), myNewRank = nextView.GetMyRank();
            int myOldAg = GetAffinityGroup(myOldRank), myNewAg = GetAffinityGroup(myNewRank), myRankInMyAg = myNewRank / myDHTnShards;
            if (myOldRank == -1)
                return;
            if (!shuffleMode && nextView.leavers.Length > 0)
            {
                // DHT state reconciliation for existing members that got shifted around by the new view and are no longer in the same affinity group
                // Each member looks for the next DHTRedundancyFactor guys who are now new members of his affinity group, but previously weren't, and sends state
                int rankOfMember = myOldRank, fnd = 0, coverFor = 0;
                for (int cnt = 1; cnt < myDHTnShards && (cnt - coverFor - 1) < DHTRedundancyFactor; cnt++)
                    if (nextView.leavers.Contains(theView.members[(myOldRank + cnt * myDHTnShards) % theView.members.Length]))
                        ++coverFor;
                for (int cnt = 0; cnt < theView.members.Length - 1 && fnd < DHTRedundancyFactor + coverFor; cnt++)
                {
                    rankOfMember = (rankOfMember + 1) % theView.members.Length;
                    Address who = theView.members[rankOfMember];
                    int rankInNextView = nextView.GetRankOf(who);
                    if (rankInNextView == -1 || GetAffinityGroup(rankInNextView) != myOldAg || GetAffinityGroup(rankInNextView) == GetAffinityGroup(rankOfMember))
                        continue;
                    else
                    {
                        // We found someone who will be in "our" affinity group but isn't currently there
                        ++fnd;
                        List<KeyValuePair<object, DHTItem>> kvps = new List<KeyValuePair<object, DHTItem>>();
                        using (new LockAndElevate(DHTDictLock))
                            foreach (var item in DHTContents)
                                kvps.Add(item);
                        if (kvps.Count() > 0)
                            P2PSend(who, Isis.IM_DHT_PUT, kvps, myNewAg, 0, nextView.viewid);
                    }
                }
                Flush();
            }
            else if (shuffleMode && myRankInMyAg < DHTRedundancyFactor)
            {
                List<KeyValuePair<object, DHTItem>>[] kvpsl = new List<KeyValuePair<object, DHTItem>>[myDHTnShards];
                for (int n = 0; n < myDHTnShards; n++)
                    kvpsl[n] = new List<KeyValuePair<object, DHTItem>>();
                using (new LockAndElevate(DHTDictLock))
                {
                    foreach (var item in DHTContents)
                        kvpsl[GetAffinityGroup(DHTKeyHash(item.Key))].Add(item);
                    for (int n = 0; n < nextView.members.Length; n++)
                        if (GetAffinityGroup(n) != myNewAg)
                        {
                            List<KeyValuePair<object, DHTItem>> kvps = kvpsl[GetAffinityGroup(n)];
                            if (kvps.Count() > 0)
                                P2PSend(nextView.members[n], Isis.IM_DHT_PUT, kvps, myNewAg, versionId, nextView.viewid);
                            else
                                P2PSend(nextView.members[n], Isis.IM_DHT_PUT, myNewAg, versionId, nextView.viewid);
                        }
                }
            }
            if (GetAffinityGroup(myOldRank) != GetAffinityGroup(myNewRank) || shuffleMode)
                using (new LockAndElevate(DHTDictLock))
                {
                    Dictionary<object, DHTItem> old = DHTContents;
                    DHTContents = new Dictionary<object, DHTItem>(1000);
                    foreach (var item in old)
                        if (GetAffinityGroup(DHTKeyHash(item.Key)) == GetAffinityGroup(myNewRank))
                            DHTContents.Add(item.Key, item.Value);
                }
        }

        private void DHTCleanup()
        {
            List<object> toDelete = new List<object>();
            long now = Isis.CurTOD();
            using (new LockAndElevate(DHTDictLock))
            {
                foreach (var kvp in DHTContents)
                    if (now - kvp.Value.createTime > myDHTItemTimeout)
                        toDelete.Add(kvp.Key);
                foreach (var key in toDelete)
                    DHTContents.Remove(key);
            }
            if (!usesDHTDefaults)
                foreach (var key in toDelete)
                    DHTExtWriter(key, null);
            Isis.OnTimer(myDHTItemTimeout, (TimerCallback)DHTCleanup);
        }

        private int GetAffinityGroup(Address a)
        {
            using (new LockAndElevate(ViewLock))
                return (theView == null) ? -1 : GetAffinityGroup(theView, a);
        }

        internal int GetAffinityGroup(View v, Address a)
        {
            return v.GetRankOf(a) % myDHTnShards;
        }

        internal int GetAffinityGroup(int hash)
        {
            return (int)(Math.Abs(hash) % myDHTnShards);
        }

        internal int DHTKeyHash(object key)
        {
            return ((key is int) ? (int)key : key.GetHashCode()) % myDHTnShards;
        }

        internal List<Address> GetAgMembers(List<object> keys)
        {
            List<Address> mlist = new List<Address>();
            bool[] IncludeAg = new bool[myDHTnShards];
            foreach (var key in keys)
                IncludeAg[GetAffinityGroup(DHTKeyHash(key))] = true;
            View theView;
            using (new LockAndElevate(ViewLock))
                theView = this.theView;
            for (int n = 0; n < theView.members.Length; n++)
                if (IncludeAg[GetAffinityGroup(n)])
                    mlist.Add(theView.members[n]);
            return mlist;
        }

        // Create something that looks like the group address but indicates the AffinityGroup via the port numbers
        internal Address PseudoAddress(int AffinityGroup)
        {
            Address pa = gaddr;
            pa.p2pPort = pa.ackPort = AffinityGroup;
            pa.cachedHashCode = 0;
            return pa;
        }

        internal int GetAffinityGrpIdx(Address a)
        {
            if (a.Equals(Isis.ISISMEMBERS.gaddr))
                return -1;
            int ag = a.p2pPort;
            a.p2pPort = a.ackPort = a.cachedHashCode = 0;
            Address mg = gaddr;
            mg.p2pPort = mg.ackPort = mg.cachedHashCode = 0;
            if (mg.Equals(a))
                return ag;
            return -1;
        }

        /// <summary>
        /// Uses the current group as a DHT and stores a new (key,value) pair, which overwrites any previous one.  
        /// </summary>
        /// <param name="key">key for the object being stored</param>
        /// <param name="value">value of that object</param>
        /// <remarks>DHT put operations aren't totally ordered, hence concurrent Add requests for the same key leave the DHT in an inconsistent state.</remarks>
        public void DHTPut(object key, object value)
        {
            DHTPut(new List<KeyValuePair<object, object>> { new KeyValuePair<object, object>(key, value) });
        }

        internal myHandlers theResolvers = new myHandlers();

        /// <summary>
        /// Polymorphic variant of SetDHTPutCollisionResolver.  
        /// </summary>
        /// <typeparam name="KT">Key type</typeparam>
        /// <typeparam name="VT">Value type</typeparam>
        /// <typeparam name="VTIncoming">Value type for the incoming value that triggered the Put collision</typeparam>
        /// <param name="theResolver">Method with signature VT theResolver(KT,VT,VTIncoming)</param>
        /// <remarks>
        /// This variant of SetDHTPutCollisionResolver allows you to specify multiple resolvers, each with
        /// its own type signature.  If you use this option, Isis2 will call the first resolver that matches.
        /// If none matches, Isis2 will check to see if there is a universal resolver for the group.
        /// </remarks>
        public void SetDHTPutCollisionResolver<KT, VT, VTIncoming>(Func<KT, VT, VTIncoming, VT> theResolver)
        {
            if (typeof(KT) == typeof(object) && typeof(VT) == typeof(object) && typeof(VTIncoming) == typeof(object))
            {
                myResolver = (dynamic)theResolver;
                return;
            }
            theResolvers += theResolver;
        }

        /// <summary>
        /// DHTPut for a single key,value pair
        /// </summary>
        /// <typeparam name="KT">Key type</typeparam>
        /// <typeparam name="VT">Value type</typeparam>
        /// <param name="key">key</param>
        /// <param name="value">value</param>
        public void DHTPut<KT, VT>(KT key, VT value)
        {
            DHTPut<KT, VT>(new List<KeyValuePair<KT, VT>>() { new KeyValuePair<KT, VT>(key, value) });
        }

        /// <summary>
        /// Uses the current group as a DHT and stores a set of new (key,value) pairs, which overwrite any previous ones.   Atomicity is not guaranteed 
        /// </summary>
        /// <param name="kvps">A list of key-value pairs to put</param>
        /// <typeparam name="KT"></typeparam>
        /// <typeparam name="VT"></typeparam>
        /// <remarks>DHT put operations aren't totally ordered, hence concurrent Add requests for the same key leave the DHT in an inconsistent state.</remarks>
        public void DHTPut<KT, VT>(List<KeyValuePair<KT, VT>> kvps)
        {
            if (!IsisCallStart())
                return;
            if (myDHTBinSize == 0)
                throw new IsisException("DHTPut: must first call DHTEnable");
            if (DHTAgNonEmpty == null)
                throw new IsisException("DHTPut: group must be created and initial members should join before first call to DHTPut");
            if (!HasFirstView || theView.members.Length < myDHTMinSize)
                throw new IsisDHTException("DHTPut: group is currently smaller than the minimum required size");
            foreach (var kvp in kvps)
            {
                object key = kvp.Key, value = kvp.Value;
                int khash = DHTKeyHash(key);
                FlowControl.FCBarrierCheck();
                int targetAg = GetAffinityGroup(khash);
                if (targetAg < 0 || targetAg > DHTAgNonEmpty.Length)
                    throw new IsisException("DHTPut: key " + key.ToString() + " mapped to affinity group " + targetAg + " which was outside range of DHTAgNonEmpty[0.." + DHTAgNonEmpty.Length + "]");
                if (!DHTAgNonEmpty[targetAg])
                {
                    Isis.WriteLine("WARNING: DHTPut called on a <key,value> pair that maps to a depopulated affinity group (hint: maybe the underlying group or the replication factor is too small!)");
                    continue;
                }
                if ((IsisSystem.Debug & IsisSystem.DHTS) != 0)
                    Isis.WriteLine("Application called DHTPut with key=" + key + ", value=" + value);
                try
                {
                    Wedged.WaitOne();
                    if (myDHTBinSize < 2 * log2(Isis.ISISMEMBERS.theView.members.Length) || myDHTInDebugMode)
                    {
                        // Until the actual size of an affinity group gets fairly large, send p2p.  But as the number of p2p sends pushes beyond log2, tunnelling will eventually
                        // be preferable despite its higher latency
                        for (int off = targetAg; off < Isis.ISISMEMBERS.theView.members.Length; off += myDHTnShards)
                            P2PSend(Isis.ISISMEMBERS.theView.members[off], Isis.IM_DHT_PUT, new List<KeyValuePair<object, DHTItem>>() { new KeyValuePair<object, DHTItem>(kvp.Key, new DHTItem(kvp.Value)) });
                    }
                    else
                        // We're use the PseudoAddress here as a kind of trick to avoid actually creating N/ReplicationFactor real groups
                        IPMCTunnel(PseudoAddress(GetAffinityGroup(khash)), Msg.toBArray(new object[] { Isis.IM_DHT_PUT, new List<KeyValuePair<object, DHTItem>>() { new KeyValuePair<object, DHTItem>(kvp.Key, new DHTItem(kvp.Value)) } }));
                }
                finally
                {
                    Wedged.Release(1);
                }
            }
            IsisCallDone();
        }

        /// <summary>
        /// Treats the the current group as a DHT and retrieves an object by key
        /// </summary>
        /// <param name="key"></param>
        /// <returns>The value from the (key,value) pair</returns>
        public VT DHTGet<KT, VT>(KT key)
        {
            var res = DHTGet<KT, VT>(new List<KT>() { key });
            if (res.Count() > 0)
                return res[0].Value;
            return default(VT);
        }

        internal static Timeout DHTTimeout = new Timeout(15000, Timeout.TO_NULLREPLY);

        /// <summary>
        /// Treats the the current group as a DHT and retrieves a list of objects by key.  No promise of atomicity
        /// with respect to DHTPut (if that's what you need, use DHTOrderedGet)
        /// </summary>
        /// <param name="ikeys"></param>
        /// <typeparam name="KT"></typeparam>
        /// <typeparam name="VT"></typeparam>
        /// <returns>A list of (key,value) pairs for items that were found</returns>
        public List<KeyValuePair<KT, VT>> DHTGet<KT, VT>(IEnumerable ikeys)
        {
            if (!IsisCallStart())
                return new List<KeyValuePair<KT, VT>>();
            View theView;
            using (new LockAndElevate(ViewLock))
                theView = this.theView;
            List<object> keys = new List<object>();
            foreach (object ik in ikeys)
                keys.Add(ik);
            if ((IsisSystem.Debug & IsisSystem.DHTS) != 0)
            {
                string ks = " ";
                foreach (var key in keys)
                    ks += key + " ";
                Isis.WriteLine("Application called DHTGet with keys { " + ks + "}");
            }
            if (myDHTBinSize == 0)
                throw new IsisException("DHTGet: must first call DHTEnable");
            if (DHTAgNonEmpty == null)
                throw new IsisException("DHTGet: group must be created and initial members should join before first call to DHTPut");
            if (!HasFirstView || GetSize() < myDHTMinSize)
                throw new IsisDHTException("DHTGet: group is currently smaller than the minimum required size");
            List<List<KeyValuePair<KT, VT>>> partialResults = new List<List<KeyValuePair<KT, VT>>>();
            var keysGroupedByAg = keys.GroupBy(key => GetAffinityGroup(DHTKeyHash(key)));
            foreach (var GroupedKey in keysGroupedByAg)
            {
                if (myDHTBinSize == 0)
                    throw new IsisException("DHTGet: must first call DHTEnable");
                int N = theView.members.Length, myBase = theView.GetMyRank(), startAt = rand.Next(N), log2N = log2(N), kgrp = GroupedKey.Key;
                Address whoToAsk = null;
                Address dontAsk = null;
                // Note: this logic only makes sense for at most 2 tries!
                for (int retry = 0; retry < 2 && dontAsk != Isis.my_address; retry++)
                {
                    try
                    {
                        if (GetAffinityGroup(myBase) == kgrp)
                            whoToAsk = Isis.my_address;
                        else
                        {
                            // First check and see if there happens to be a 1-hop neighbor we could ask
                            for (int n = 0; whoToAsk == null && n < log2N; n++)
                                if (GetAffinityGroup((myBase + (1 << n)) % N) == kgrp && (dontAsk == null || !theView.members[(myBase + (1 << n)) % N].Equals(dontAsk)))
                                    whoToAsk = theView.members[(myBase + (1 << n)) % N];
                        }

                        // If no luck, check starting at some random location and just take anyone in the right affinity group
                        for (int n = 0; whoToAsk == null && n < theView.members.Length; n++)
                            if (GetAffinityGroup((startAt + n) % N) == kgrp && (dontAsk == null || !theView.members[(startAt + n) % N].Equals(dontAsk)))
                                whoToAsk = theView.members[(startAt + n) % N];

                        if (whoToAsk == null)
                            throw new IsisException("DHT can't find anyone to ask about keys mapping to DHT affinity group " + kgrp);
                        List<KT> gk = new List<KT>();
                        foreach (KT grouped in GroupedKey)
                            gk.Add(grouped);
                        try
                        {
                            Wedged.WaitOne();
                            if (!P2PQuery(whoToAsk, DHTTimeout, Isis.IM_DHT_GET, gk, EOL, partialResults))
                            {
                                // Two cases: He may have done a nullreply, or he may have actually failed.
                                using (new LockAndElevate(ViewLock))
                                    theView = this.theView;
                                if (theView.GetRankOf(whoToAsk) == -1)
                                    dontAsk = whoToAsk;
                                continue;
                            }
                        }
                        finally
                        {
                            Wedged.Release(1);
                        }
                    }
                    catch (IsisAbortReplyException)
                    {
                        Isis.WriteLine("DHTGet: AbortReplyException!");
                        dontAsk = whoToAsk;
                    }
                }
            }
            Dictionary<KT, VT> deDuped = new Dictionary<KT, VT>(1000);
            List<KeyValuePair<KT, VT>> results = new List<KeyValuePair<KT, VT>>();
            foreach (var kvpl in partialResults)
                foreach (var res in kvpl)
                    if (!deDuped.ContainsKey(res.Key))
                        deDuped.Add(res.Key, res.Value);
            foreach (var res in deDuped)
                results.Add(res);
            IsisCallDone();
            return results;
        }

        /// <summary>
        /// Atomically and with total order, puts a key,value pair into the DHT
        /// </summary>
        /// <param name="key"></param>
        /// <param name="value"></param>
        /// <typeparam name="KT"></typeparam>
        /// <typeparam name="VT"></typeparam>
        public void DHTOrderedPut<KT, VT>(KT key, VT value)
        {
            if (!IsisCallStart())
                return;
            OrderedSend(GetAgMembers(new List<object>() { key }), Isis.IM_DHT_PUT, new List<KeyValuePair<object, DHTItem>>() { new KeyValuePair<object, DHTItem>(key, new DHTItem(value)) });
            IsisCallDone();
        }

        /// <summary>
        /// Atomically and with total order, puts a set of key,value pairs into the DHT
        /// </summary>
        /// <param name="kvps"></param>
        /// <typeparam name="KT"></typeparam>
        /// <typeparam name="VT"></typeparam>
        public void DHTOrderedPut<KT, VT>(List<KeyValuePair<KT, VT>> kvps)
        {
            if (!IsisCallStart())
                return;
            List<KeyValuePair<KT, DHTItem>> newList = new List<KeyValuePair<KT, DHTItem>>();
            foreach (var kvp in kvps)
                newList.Add(new KeyValuePair<KT, DHTItem>(kvp.Key, new DHTItem(kvp.Value)));
            if (newList.Count() > 0)
                OrderedSend(GetAgMembers(kvps.Select(kvp => (object)kvp.Key).ToList<object>()), Isis.IM_DHT_PUT, newList);
            IsisCallDone();
        }

        /// <summary>
        /// Atomically and with total order, removes a set of keys from the DHT
        /// </summary>
        /// <param name="keys"></param>
        /// <typeparam name="KT"></typeparam>
        public void DHTOrderedRemove<KT>(IEnumerable<KT> keys)
        {
            if (!IsisCallStart())
                return;
            byte[] b0 = new byte[0];
            List<KeyValuePair<KT, byte[]>> toSend = new List<KeyValuePair<KT, byte[]>>();
            foreach (KT key in keys)
                toSend.Add(new KeyValuePair<KT, byte[]>(key, b0));
            if (toSend.Count() > 0)
                OrderedSend(GetAgMembers(keys.Select(k => (object)k).ToList<object>()), Isis.IM_DHT_PUT, toSend);
            IsisCallDone();
        }

        /// <summary>
        /// Removes the value associated with some set of keys.  Atomicity is not guaranteed
        /// </summary>
        /// <param name="keys"></param>
        /// <typeparam name="KT"></typeparam>
        /// <returns>The value from the (key,value) pair</returns>
        /// <remarks>DHT operations are reliable but not totally ordered, hence DHTRemove for a key shouldn't be issued concurrently with DHTPut operations using the identical key.</remarks>
        public void DHTRemove<KT>(IEnumerable<KT> keys)
        {
            if (!IsisCallStart())
                return;
            foreach (var key in keys)
                DHTPut(key, new byte[0]);
            IsisCallDone();
        }

        /// <summary>
        /// Removes the value associated with the key.
        /// </summary>
        /// <param name="key"></param>
        /// <typeparam name="KT"></typeparam>
        /// <returns>The value from the (key,value) pair</returns>
        /// <remarks>DHT operations are reliable but not totally ordered, hence DHTRemove for a key shouldn't be issued concurrently with DHTPut operations using the identical key.</remarks>
        public void DHTRemove<KT>(KT key)
        {
            if (!IsisCallStart())
                return;
            DHTPut(key, new byte[0]);
            IsisCallDone();
        }

        /// <summary>
        /// Performs a virtually synchronous DHTGet operation for a set of keys, returning a list of (key,value) pairs, deduplicated by key
        /// </summary>
        /// <param name="keys"></param>
        /// <typeparam name="KT"></typeparam>
        /// <typeparam name="VT"></typeparam>
        /// <returns></returns>
        public List<KeyValuePair<KT, VT>> DHTOrderedGet<KT, VT>(IEnumerable<KT> keys)
        {
            if (!IsisCallStart())
                return new List<KeyValuePair<KT, VT>>();
            List<List<KeyValuePair<KT, VT>>> results = new List<List<KeyValuePair<KT, VT>>>();
            List<KT> toSend = new List<KT>();
            foreach (KT key in keys)
                toSend.Add(key);
            OrderedQuery(ALL, DHTTimeout, new QueryKey<KT>(Isis.my_address, keys, false), Isis.IM_DHT_GET, toSend, EOL, results);
            Dictionary<KT, VT> deDuped = new Dictionary<KT, VT>(1000);
            foreach (var rl in results)
                foreach (var res in rl)
                    if (!deDuped.ContainsKey(res.Key))
                        deDuped.Add(res.Key, res.Value);
            List<KeyValuePair<KT, VT>> rv = new List<KeyValuePair<KT, VT>>();
            foreach (var res in deDuped)
                rv.Add(res);
            IsisCallDone();
            return rv;
        }

        /// <summary>
        /// Performs a DHTGet operation for a single key, returning the value or null
        /// </summary>
        /// <typeparam name="KT"></typeparam>
        /// <typeparam name="VT"></typeparam>
        /// <param name="key"></param>
        /// <returns></returns>
        /// <remarks>In this case DHTOrderedGet and DHTGet are actually identical</remarks>
        public VT DHTOrderedGet<KT, VT>(KT key)
        {
            return DHTGet<KT, VT>(key);
        }

        internal bool multicastSendIsStable(int vid, int msgid)
        {
            foreach (ReliableSender.MsgDesc md in ReliableSender.PendingSendBuffer)
                if (md.group == this && md.MsgVid == vid && md.MsgId == msgid)
                    return false;
            return true;
        }

        internal static Dictionary<Address, bool[]> hmInfo = new Dictionary<Address, bool[]>(1000);
        internal static LockObject hmLock = new LockObject("hmLock");

        internal static int log2(int N)
        {
            int i;
            for (i = 0; (1 << i) < N; i++)
                continue;
            // Rounds down
            if ((1 << i) == N)
                return i;
            return i - 1;
        }

        internal static int log2RU(int N)
        {
            int i;
            for (i = 0; (1 << i) < N; i++)
                continue;
            return i;
        }

        internal static string TunnelState()
        {
            if (!Isis.ISIS_TCP_ONLY || Isis.ISISMEMBERS == null || !Isis.ISISMEMBERS.HasFirstView)
                return "";
            string ms = "";
            View v;
            using (new LockAndElevate(IPMCNewViewLock))
                v = TunnelView;
            if (v == null)
                return "TCP_ONLY STATE PROBLEM: TunnelView shouldn't be null\r\n";
            int sl = 0, myBase = v.GetMyRank(), N = v.members.Length;
            for (int i = 1; i < v.members.Length; i <<= 1)
            {
                ++sl;
                ms += v.members[(i + myBase) % N];
            }
            using (new LockAndElevate(hmLock))
            {
                string s = "TCP_ONLY TUNNEL STATE for ISISMEMBERS... gaddr=" + Isis.ISISMEMBERS.gaddr + " Tunneling with viewid= " + v.viewid + "{" + Address.VectorToString(v.members) +
                    "}::\r\n  SkipList = [" + ms + "]; myBase=" + myBase + ", hmInfo maps " + hmInfo.Count + " overlay groups\r\n";
                foreach (KeyValuePair<Address, bool[]> kvp in hmInfo)
                    s += "   " + kvp.Key + " => {" + PHMap(kvp.Value, sl) + "}\r\n";
                return s;
            }
        }

        private static string PHMap(bool[] hm)
        {
            return PHMap(hm, int.MaxValue);
        }

        private static string PHMap(bool[] hm, int nb)
        {
            string bs = " ";
            foreach (bool b in hm)
                if (nb-- > 0)
                    bs += (b ? "+" : "-") + " ";
            return bs;
        }

        internal class ddh
        {
            internal Address dest;
            internal byte[] data;
            internal int hopcnt;

            internal ddh(Address d, byte[] da, int h)
            {
                dest = d; data = da; hopcnt = h;
            }
        }

        internal class gsdh
        {
            internal Address gaddr;
            internal Address sender;
            internal byte[] data;
            internal int hopcnt;

            internal gsdh(Address g, Address s, byte[] da, int h)
            {
                gaddr = g; sender = s; data = da; hopcnt = h;
            }
        }

        internal static LockObject TunnelThreadsLock = new LockObject("TunnelThreadsLock");
        internal static Thread IPMCTunnelThread, UDPTunnelThread;
        internal static List<ddh> ddhList = new List<ddh>();
        internal static LockObject ddhLock = new LockObject("ddhLock", ThreadPriority.AboveNormal);
        internal Semaphore ddhNotEmpty = new Semaphore(0, int.MaxValue);
        internal static List<gsdh> gsdhList = new List<gsdh>();
        internal static LockObject gsdhLock = new LockObject("gsdhLock");
        internal Semaphore gsdhNotEmpty = new Semaphore(0, int.MaxValue);

        internal static int nInUDPTunnel()
        {
            using (new LockAndElevate(ddhLock))
                return ddhList.Count();
        }

        internal static int nInIPMCTunnel()
        {
            using (new LockAndElevate(gsdhLock))
                return gsdhList.Count();
        }

        internal class IPMCVinfo
        {
            internal int IMVid;
            internal Address gaddr;
            internal Address sender;
            internal View v;

            internal IPMCVinfo(int vid, Address ga, Address s, View theView)
            {
                IMVid = vid;
                gaddr = ga;
                sender = s;
                v = theView;
            }
        }

        // These need to spawn threads because they can't risk blocking on the looped-back deliveries via the RECVB bounded buffer
        internal void SetupIMTunnels()
        {
            doRegister(Isis.BECLIENT, (Action<Address>)delegate(Address who)
            {
                becomeClientOf(who);
            });
            doRegister(Isis.IM_UDP_TUNNEL, (Action<Address, byte[], int>)delegate(Address dest, byte[] data, int hopcnt)
            {
                if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                    Isis.WriteLine("UDP Tunnel event handler: got a new UDP packet tunnelling to dest=" + dest + ", data.Length=" + data.Length + ", hopcount is " + hopcnt);
                using (new LockAndElevate(ddhLock))
                {
                    ddhList.Add(new ddh(dest, data, hopcnt));
                    ddhNotEmpty.Release(1);
                }
                using (new LockAndElevate(TunnelThreadsLock))
                {
                    if (UDPTunnelThread == null)
                    {
                        UDPTunnelThread = new Thread(delegate()
                        {
                            try
                            {
                                while (!IsisSystem.IsisActive)
                                    Thread.Sleep(250);
                                Thread.CurrentThread.Name = "IM_UDP_TUNNEL thread";
                                Thread.CurrentThread.Priority = ThreadPriority.AboveNormal;
                                ddh dh;
                                while (IsisSystem.IsisActive)
                                {
                                    IsisSystem.RTS.ThreadCntrs[6]++;
                                    ddhNotEmpty.WaitOne(1000);
                                    using (new LockAndElevate(ddhLock))
                                    {
                                        dh = ddhList.FirstOrDefault();
                                        if (dh != null)
                                            ddhList.Remove(dh);
                                    }
                                    if (dh != null)
                                    {
                                        if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                                            Isis.WriteLine("UDP Tunnel event handler calling UDPTunnel: dest=" + dh.dest + ", data.Length=" + dh.data.Length);
                                        UDPTunnel(dh.dest, dh.data, dh.hopcnt + 1);
                                    }
                                }
                            }
                            catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                            IsisSystem.ThreadTerminationMagic();
                        });
                        UDPTunnelThread.Start();
                    }
                }
            });
            doRegister(Isis.IM_IPMC_TUNNEL, (Action<Address, Address, byte[], int>)delegate(Address gaddr, Address sender, byte[] data, int hopcnt)
            {
                if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                    Isis.WriteLine("IPMC Tunnel[ql=" + gsdhList.Count() + "] got a new incoming request: gaddr=" + gaddr + ", sender=" + sender + ", data length=" + data.Length + ", hopcnt=" + hopcnt);
                using (new LockAndElevate(gsdhLock))
                    gsdhList.Add(new gsdh(gaddr, sender, data, hopcnt));
                gsdhNotEmpty.Release(1);
                using (new LockAndElevate(TunnelThreadsLock))
                {
                    if (IPMCTunnelThread == null)
                    {
                        IPMCTunnelThread = new Thread(delegate()
                        {
                            try
                            {
                                while (!IsisSystem.IsisActive)
                                    Thread.Sleep(250);
                                Thread.CurrentThread.Name = "IM_IPMC_TUNNEL thread";
                                Thread.CurrentThread.Priority = ThreadPriority.AboveNormal;
                                gsdh gdh;
                                while (IsisSystem.IsisActive)
                                {
                                    IsisSystem.RTS.ThreadCntrs[7]++;
                                    gsdhNotEmpty.WaitOne(1000);
                                    using (new LockAndElevate(gsdhLock))
                                        gdh = gsdhList.FirstOrDefault();
                                    if (gdh != null)
                                    {
                                        gsdhList.Remove(gdh);
                                        int ag = GetAffinityGrpIdx(gdh.gaddr);
                                        if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                                            Isis.WriteLine("IPMC Tunnel[ql=" + gsdhList.Count() + "]  event handler: gaddr=" + gaddr + ((ag == -1) ? "" : ("(affinity group " + ag + ")")) + ", sender=" + gdh.sender + ", data.Length=" + gdh.data.Length);
                                        IPMCNextHops(gdh.gaddr, gdh.sender, gdh.hopcnt, (Action<Address, int>)delegate(Address nextHop, int nToScan)
                                        {
                                            if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                                                Isis.WriteLine("IPMC Tunnel event handler: nextHop will be " + nextHop + " for  gaddr=" + ((ag == -1) ? gdh.gaddr.ToString() : ("(affinity group " + ag + ")")) + ", sender=" + gdh.sender + ", data.Length=" + gdh.data.Length);
                                            Isis.ISISMEMBERS.doP2PSend(nextHop, true, Isis.IM_IPMC_TUNNEL, gdh.gaddr, gdh.sender, gdh.data, nToScan);
                                        });
                                        if ((ag == -1 && Group.doLookup(gdh.gaddr) != null) || (ag >= 0 && Isis.my_address.GetHashCode() % myDHTnShards == ag))
                                            IPMCTunnelLoopback(gdh.gaddr, gdh.data);
                                    }
                                }
                            }
                            catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                            IsisSystem.ThreadTerminationMagic();
                        });
                        IPMCTunnelThread.Start();
                    }
                }
            });

            BoundedBuffer IPMCbb = new BoundedBuffer("IPMC:BB", 256, ILock.LLIPMC, 0, 1);
            new Thread(delegate()
            {
                try
                {
                    while (!IsisSystem.IsisActive)
                        Thread.Sleep(250);
                    Thread.CurrentThread.Name = "IM_IPMCViewCast_TUNNEL thread";
                    while (IsisSystem.IsisActive)
                    {
                        IsisSystem.RTS.ThreadCntrs[8]++;
                        IPMCVinfo vi = (IPMCVinfo)IPMCbb.get();
                        if (vi == null)
                            break;
                        IPMCViewCast(vi.IMVid, vi.gaddr, vi.sender, vi.v);
                    }
                }
                catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                IsisSystem.ThreadTerminationMagic();
            }).Start();

            doRegister(Isis.IM_IPMC_TUNNEL, (Action<int, Address, Address, int, View>)delegate(int IMVid, Address gaddr, Address sender, int hopcnt, View v)
            {
                if (Isis.ISISMEMBERS.HasFirstView && Isis.ISISMEMBERS.theView.viewid == IMVid)
                    IPMCbb.put(new IPMCVinfo(IMVid, gaddr, sender, v));
                else if (!Isis.ISISMEMBERS.HasFirstView || Isis.ISISMEMBERS.theView.viewid < IMVid)
                    using (new LockAndElevate(Isis.ISISMEMBERS.ViewLock))
                        Isis.ISISMEMBERS.stashedIPMCviews.Add(new IPMCVinfo(IMVid, gaddr, sender, v));
                IPMCNextHops(gaddr, sender, hopcnt, (Action<Address, int>)delegate(Address nextHop, int nToScan)
                {
                    Isis.ISISMEMBERS.doP2PSend(nextHop, true, Isis.IM_IPMC_TUNNEL, gaddr, sender, nToScan, v);
                });
            });

            doRegister(Isis.IM_IPMC_VIEWS, (Action<View[]>)delegate(View[] vs)
            {
                foreach (View v in vs)
                    IPMCNewView(v.gaddr, v);
            });
        }

        internal void replayStashedVinfo()
        {
            List<IPMCVinfo> newstash = new List<IPMCVinfo>(), toReplay = new List<IPMCVinfo>();
            View IMv;
            using (new LockAndElevate(Isis.ISISMEMBERS.ViewLock))
            {
                IMv = Isis.ISISMEMBERS.theView;
                foreach (IPMCVinfo vi in stashedIPMCviews)
                    if (vi.IMVid > IMv.viewid)
                        newstash.Add(vi);
                    else if (vi.IMVid == IMv.viewid)
                        toReplay.Add(vi);
                stashedIPMCviews = newstash;
            }
            if (toReplay.Count() > 0)
                new Thread(delegate()
                {
                    try
                    {
                        Thread.CurrentThread.Name = "ReplayStashedVinfo";
                        foreach (IPMCVinfo vi in toReplay)
                            IPMCViewCast(vi.IMVid, vi.gaddr, vi.sender, vi.v);
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    IsisSystem.ThreadTerminationMagic();
                }).Start();
        }

        // Just sends directly 
        internal void UDPTunnel(Address dest, byte[] data, int hopcnt)
        {
            Isis.ISISMEMBERS.UDPTunnel(dest, 0, data, hopcnt);
        }

        internal class ShortCutInfo
        {
            internal Address who;
            internal Address gaddr;

            internal ShortCutInfo(Address w, Address g) { who = w; gaddr = g; }
        }

        internal static List<ShortCutInfo> AllShortCuts = new List<ShortCutInfo>();
        internal static LockObject ShortCutsLock = new LockObject("ShortCutsLock", ThreadPriority.Highest);
        internal Dictionary<Address, bool> ShortCuts = new Dictionary<Address, bool>(1000);

        internal void UpdateShortCuts(View v)
        {
            List<ShortCutInfo> scl = new List<ShortCutInfo>();
            Group g = Group.doLookup(v.gaddr);
            if (g == null)
                return;
            if (g == Isis.ORACLE && Isis.ClientOf == null)
            {
                foreach (Address a in g.theView.members)
                    if (!a.isMyAddress())
                        scl.Add(new ShortCutInfo(a, g.gaddr));
            }
            else
            {
                tokenInfo theToken;
                using (new LockAndElevate(g.TokenLock))
                    theToken = g.theToken;
                if ((g.flags & Group.G_ISLARGE) != 0 && theToken != null)
                {
                    for (int level = 0; level < theToken.nlevels; level++)
                    {
                        if (theToken.next[level] != null && !theToken.next[level].isMyAddress())
                            scl.Add(new ShortCutInfo(theToken.next[level], v.gaddr));
                        if (theToken.last[level] != null && !theToken.last[level].isMyAddress())
                            scl.Add(new ShortCutInfo(theToken.last[level], v.gaddr));
                    }
                }
            }
            using (new LockAndElevate(ShortCutsLock))
            {
                ShortCuts = new Dictionary<Address, bool>(1000);
                foreach (ShortCutInfo sci in scl)
                    if (!ShortCuts.ContainsKey(sci.who))
                        ShortCuts.Add(sci.who, true);
                foreach (ShortCutInfo sci in AllShortCuts)
                    if (!sci.gaddr.Equals(v.gaddr))
                        scl.Add(sci);
                AllShortCuts = scl;
            }
        }

        internal static bool notNeeded(Address dest)
        {
            if (Isis.IAmOracle && Isis.ORACLE.IAmLeader() && (Isis.ISISMEMBERS.theToken == null || Isis.ISISMEMBERS.theToken.WorkingView == null || Isis.ISISMEMBERS.theToken.WorkingView.GetRankOf(dest) == -1))
                return false;
            if (ReliableSender.HeardFromRecently(dest) || IsisSystem.IsisRestarting)
                return false;
            using (new LockAndElevate(ShortCutsLock))
            {
                foreach (ShortCutInfo si in AllShortCuts)
                    if (si.who.Equals(dest))
                        return false;
            }
            List<Group> ag = IsisAllGroupsClone(true);
            foreach (Group g in ag)
                if (!g.isTrackingProxy && ((g.flags & G_ISLARGE) != 0) || g == Isis.ISISMEMBERS)
                {
                    View theView;
                    if ((g.flags & G_ISLARGE) == 0)
                        using (new LockAndElevate(g.ViewLock))
                            theView = g.theView;
                    else using (new LockAndElevate(g.TokenLock))
                            if (g.theToken == null)
                                continue;
                            else
                                theView = g.theToken.WorkingView;
                    int myRank = theView.GetMyRank(), N = theView.members.Length;
                    for (int m = 1; m < theView.members.Length; m <<= 1)
                        if (dest.Equals(theView.members[(m + myRank) % N]))
                            return false;
                }
            return true;
        }


        // Sends but "jiggers" the route for retransmission attempts
        internal void UDPTunnel(Address dest, int jigger, byte[] data, int hopcnt)
        {
            if (hopcnt > 10)
            {
                if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                    Isis.WriteLine("WARNING: Discarding a packet in UDPTunnel: hopcnt>10");
                using (new LockAndElevate(IsisSystem.RTS.Lock))
                    ++IsisSystem.RTS.Discarded;
                return;
            }
            View v = null;
            if (!Isis.ISIS_LARGE)
                using (new LockAndElevate(Isis.ISISMEMBERS.ViewLock))
                    v = Isis.ISISMEMBERS.theView;
            else using (new LockAndElevate(Isis.ISISMEMBERS.TokenLock))
                    if (Isis.ISISMEMBERS.theToken != null)
                        v = Isis.ISISMEMBERS.theToken.WorkingView;
            if (v == null)
            {
                if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                    Isis.WriteLine("WARNING: Discarding a packet in UDPTunnel: the working view is null");
                using (new LockAndElevate(IsisSystem.RTS.Lock))
                    ++IsisSystem.RTS.Discarded;
                return;
            }
            if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                Isis.WriteLine("UDP Tunnel sender method: dest=" + dest + ", jigger=" + jigger + " data length=" + data.Length);
            if (dest.isMyAddress())
            {
                if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                    Isis.WriteLine("UDP Tunnel: I'm the destination! loop back");
                IPMCTunnelLoopback(gaddr, data);
                return;
            }
            if (Isis.ISIS_TCP_ONLY && ReliableSender.TCPSendIfKnown(dest, data, ReliableSender.RECVBB))
            {
                if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                    Isis.WriteLine("UDP Tunnel: dest " + dest + " was known, sent directly");
                return;
            }
            int mybaseRank = v.GetMyRank(), destRank = v.GetRankOf(dest), N = v.members.Length;
            if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                Isis.WriteLine("MybaseRank=" + mybaseRank + ", destRank=" + destRank + ", N=" + N);
            // This next line is the core of the UDP emulation layer, and what we do is to forward the packet
            // in hops.  Start by assuming jigger is always 0.
            //
            // For example to reach the member with rank 5 from a sender who happens to have rank 0 in a group of size 9,
            // this code computes log2((9-0+5) mod 9), which is log2(5), hence 2.  Call this k.  The code then 
            // then forwards it to member (1<<k), which is member 4.
            //
            // Member 4 receives it, and it computes log2((9-4+5)%9), which is log2(1), which is k=0.  So member 4
            // forwards it member 5.
            // 
            // Now think about the same scenario but sending from 5 back to 0, same group of size 9
            //    First we compute (9-5+0)%9, and we end up with 4.
            //    Now we compute log2(4)=2, then 1<<2, which is 4 again.
            //    Add 5 back in and we have 9, mod 9, giving 0.  So we forward directly from member 5 to member 0.
            // 
            // This has the effect of maintaining log(N) links from each node in ISISMEMBERS to the nodes at offsets N/2, N/4, N/8, etc to the "right",
            // wrapping as we get to N.  So we need log(N) TCP links and can reach all N members in log(N) hops, worst case
            //
            // The "jigger" parameter is used when retransmitting UDP packets to avoid having a single failure "disconnect" the whole subtree of nodes below
            // the failed node.  When retransmitting, this forces us to use the "wrong" first hop if non-zero.  By virtue of that the second and subsequent messages
            // still follow an optimal route, but it isn't always the same route.  To do this we just change which of the log(N) outgoing links we use for the
            // first hop, unless the first hop would have taken us directly to the target node.
            //
            // So... the "trueTargetOffset" looks at how far the actual destination is to the "right" of me in the view
            int trueTargetOffset = (N - mybaseRank + destRank) % N;
            // The base2 log of the offset rounds down and determines which TCP overlay link we'll use: N/2, N/4, ... 0
            int l2OfTrueOffset = log2(trueTargetOffset);
            // That TCP overlay link takes us to some node, call it the "selected" target.  Compute it's offset to the right of me
            int selectedTargetOffset = 1 << l2OfTrueOffset;
            // If we're retransmitting and the target isn't the next hop...
            if (jigger != 0 && selectedTargetOffset != trueTargetOffset)
                // Carefully apply the jigger, making sure we don't accidently exceed log2(N)
                selectedTargetOffset = 1 << ((l2OfTrueOffset + jigger) % (log2(N) + 1));
            // Now apply the selected target offset to my own base rank, mod N, and that gives the node to which we'll send the UDP packet
            bool isShortCut = false;
            using (new LockAndElevate(ShortCutsLock))
                ShortCuts.TryGetValue(dest, out isShortCut);
            if (mybaseRank == -1 || destRank == -1 || trueTargetOffset == (1 << l2OfTrueOffset) || isShortCut)
            {
                if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                    Isis.WriteLine("UDPTunnel... ReliableSender.P2PSend: dest=" + dest + ", mybaseRank=" + mybaseRank + ", destRank=" + destRank + ", using direct TCPSendTo");
                ReliableSender.P2PSend(dest, dest.p2pPort, data, ReliableSender.RECVBB);
                return;
            }
            if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                Isis.WriteLine("UDPTunnel.... ReliableSender.doP2PSend: dest= " + dest + "... computed target " + v.members[(mybaseRank + selectedTargetOffset) % N] + ", using mybaseRank=" + mybaseRank + ", destRank=" + destRank + ", N=" + N +
                    ", trueTargetOffset=" + trueTargetOffset + ", l2ofTrueTargetOffset=" + l2OfTrueOffset + ", selectedTargetOffset=" + selectedTargetOffset);
            Address target = v.members[(mybaseRank + selectedTargetOffset) % N];
            doP2PSend(target, true, Isis.IM_UDP_TUNNEL, dest, data, hopcnt);
        }

        internal void IPMCTunnel(Address gaddr, byte[] data)
        {
            if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                Isis.WriteLine("IPMC Tunnel sender method: gaddr=" + gaddr + ", data.Length=" + data.Length);
            if (Group.doLookup(gaddr) != null)
                IPMCTunnelLoopback(gaddr, data);
            IPMCNextHops(gaddr, Isis.my_address, Int32.MaxValue, (Action<Address, int>)delegate(Address nextHop, int nToScan)
            {
                if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                    Isis.WriteLine("IPMC Tunnel next hop will be via " + nextHop + " for message to gaddr=" + gaddr + ", data.Length=" + data.Length);
                Isis.ISISMEMBERS.doP2PSend(nextHop, true, Isis.IM_IPMC_TUNNEL, gaddr, Isis.my_address, data, nToScan);
            });
        }

        private static void IPMCTunnelLoopback(Address gaddr, byte[] data)
        {
            ReliableSender.AddToReceiveBB(data, true);
        }

        internal static void IPMCViewCast(int IMVid, Address gaddr, Address sender, View v)
        {
            View IMv = null;
            if (!Isis.ISIS_LARGE)
                using (new LockAndElevate(Isis.ISISMEMBERS.ViewLock))
                    IMv = Isis.ISISMEMBERS.theView;
            else using (new LockAndElevate(Isis.ISISMEMBERS.TokenLock))
                    if (Isis.ISISMEMBERS.theToken != null)
                        IMv = Isis.ISISMEMBERS.theToken.WorkingView;
            if (IMv == null || IMv.viewid != IMVid)
                return;
            if (gaddr.Equals(Isis.ISISMEMBERS.gaddr))
            {
                // When membership changes, we need to reset all the forwarding tables because the algorithm is dependent on the view of ISISMEMBERS
                if (Isis.ClientOf == null && Isis.ORACLE.HasFirstView && Isis.ORACLE.theView.GetMyRank() == 0)
                {
                    List<View> theViews = new List<View>();
                    using (new LockAndElevate(Group.TPGroupsLock))
                        foreach (var kvp in Group.TPGroups)
                        {
                            Group tpg = kvp.Value;
                            if (tpg.theView != null && !tpg.gaddr.Equals(Isis.ISISMEMBERS.gaddr))
                                theViews.Add(tpg.theView);
                        }
                    Isis.ISISMEMBERS.doSend(false, false, Isis.IM_IPMC_VIEWS, theViews.ToArray());
                }
                Isis.ISISMEMBERS.replayStashedVinfo();
                return;
            }
            // This case is used when some single group has a membership change.  Logic requires that these P2P messages be delivered after the IM_IPMC_VIEWS multicast
            // associated with the most recent ISISMEMBERS membership update.
            if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                Isis.WriteLine("IPMCViewCast[" + Isis.my_address + "]: gaddr=" + gaddr + ", sender=" + sender + ", View=" + v);
            IPMCNewView(gaddr, v);
            IPMCNextHops(gaddr, sender, Int32.MaxValue, (Action<Address, int>)delegate(Address nextHop, int nToScan)
            {
                if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                    Isis.WriteLine("IPMCViewCast[" + Isis.my_address + "] next hop will be via " + nextHop + " for gaddr=" + gaddr + ", sender=" + sender + ", View=" + v);
                if (nextHop.Equals(sender))
                    throw new IsisException("IPMCNextHops: trying to forward back to the originator of a multicast!");
                Isis.ISISMEMBERS.doP2PSend(nextHop, true, Isis.IM_IPMC_TUNNEL, IMv.viewid, gaddr, sender, nToScan, v);
            });
        }

        // This guy does a callback to the designated action routine for each next hop the data should take
        internal static void IPMCNextHops(Address gaddr, Address sender, int nToScan, Action<Address, int> action)
        {
            if (nToScan > 20)
            {
                if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                    Isis.WriteLine("WARNING: Discarding a packet in UDPTunnel: nToScan>20");
                using (new LockAndElevate(IsisSystem.RTS.Lock))
                    ++IsisSystem.RTS.Discarded;
                return;
            }
            if (!Isis.ISISMEMBERS.HasFirstView)
                return;
            View v = null;
            using (new LockAndElevate(Isis.ISISMEMBERS.TokenLock))
                if (Isis.ISISMEMBERS.theToken != null)
                    v = Isis.ISISMEMBERS.theToken.WorkingView;
            if (v == null)
            {
                if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                    Isis.WriteLine("WARNING: Discarding a packet in IPMCNextHops: the working view is null");
                using (new LockAndElevate(IsisSystem.RTS.Lock))
                    ++IsisSystem.RTS.Discarded;
                return;
            }
            int myBase = v.GetMyRank(), sbase = v.GetRankOf(sender);
            int N = v.members.Length, ag = Isis.ISISMEMBERS.GetAffinityGrpIdx(gaddr);
            if (sbase <= myBase)
                sbase += N;
            bool[] hm = null, hmm = null;
            if (gaddr.Equals(Isis.ISISMEMBERS.gaddr))
            {
                hm = new bool[log2(v.members.Length) + 1];
                // With each hop deeper into the tree, halve the link span that we'll look at
                if (nToScan > hm.Length)
                    nToScan = hm.Length;
                for (int b = 0; b < nToScan; b++)
                    hm[b] = true;
                if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                    Isis.WriteLine("IPMC Tunnel: dest is ISISMEMBERS, setting hm = {" + PHMap(hm) + "}");
            }
            else if (ag >= 0)
            {
                hm = new bool[Isis.ISISMEMBERS.myDHTnShards];
                if (nToScan > hm.Length)
                    nToScan = hm.Length;
                for (int b = 0; b < nToScan; b++)
                    hm[b] = Isis.ISISMEMBERS.DHTHMaps[ag, b];
                if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                    Isis.WriteLine("IPMC Tunnel: is an artificial DHT affinity subgroup with hm = {" + PHMap(hm) + "}");
            }
            else
            {
                using (new LockAndElevate(hmLock))
                    hmInfo.TryGetValue(gaddr, out hmm);
                if (hmm != null)
                {
                    hm = new bool[hmm.Length];
                    if (nToScan > hm.Length)
                        nToScan = hm.Length;
                    for (int b = 0; b < nToScan; b++)
                        hm[b] = hmm[b];
                }
            }
            if (hm == null || myBase == -1)
            {
                if ((IsisSystem.Debug & (IsisSystem.TUNNELING | IsisSystem.DISCARDS)) != 0)
                    Isis.WriteLine("WARNING: IPMC tunnel ignoring a packet with gaddr=" + gaddr + ", sender=" + sender + ", pseudoDepth=" + nToScan + " (couldn't find the group, or my own rank)");
                using (new LockAndElevate(IsisSystem.RTS.Lock))
                    IsisSystem.RTS.Discarded++;
                return;
            }
            // The idea below is pretty simple, but sometimes simple things are hard to write down.  Suppose that my rank is 0 and I want to broadcast.
            // I would send a packet on every link I'm allowed to use, namely to the member at 1, 2, ... N/2
            // Notice that this would be log2(N) packets sent.  But to whom should each receiver forward?
            //
            // At the next "depth" in the forwarding tree, the guy I sent to via my first link out shouldn't forward: he doesn't look at "any" links
            // The guy I reached via my second link out should forward only on his own first link: he looks at link 0
            // The guy I reached via my third link out should forward on on his first two links, etc.
            //
            // Next level down repeats this idea
            // So we get a kind of tree: shallow on its left, log2(N) deep if we follow its extreme right side to the bottom
            // Any given multicast always tracks down the binary tree that we saw "looking down" from its sender (the root)
            //
            // Surprisingly, the actual rule used is pretty simple, once you visualize it this way
            //
            if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
            {
                string slist = " ";
                for (int i = 1; i < v.members.Length; i <<= 1)
                    slist += v.members[(myBase + i) % N];
                Isis.WriteLine("IPMC Tunnel: gaddr=" + gaddr + ", sender=" + sender + ", myBase=" + myBase + ", sbase=" + sbase + ", nToScan=" + nToScan +
                    ", v.members=" + Address.VectorToString(v.members) + ", hm = {" + PHMap(hm) + "} in skiplist= {" + slist + "}");
            }
            for (int k = 0; k < hm.Length; k++)
                if (hm[k])
                {
                    if ((myBase + (1 << k)) < sbase)
                    {
                        if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                            Isis.WriteLine("IPMC Tunnel: Sending a copy to v.members[" + (1 << k) + "]=" + v.members[(myBase + (1 << k)) % N]);
                        action(v.members[(myBase + (1 << k)) % N], k);
                    }
                }
        }

        internal static LockObject IPMCNewViewLock = new LockObject("IPMCNewViewLock");
        internal static View TunnelView;
        internal static void IPMCNewView(Address gaddr, View v)
        {
            using (new LockAndElevate(IPMCNewViewLock))
                if (gaddr.Equals(Isis.ISISMEMBERS.gaddr))
                {
                    if (TunnelView != null && v.viewid <= TunnelView.viewid)
                        return;
                    TunnelView = v;
                    // ISISMEMBERS view changed: recompute everything
                    Dictionary<Address, bool[]> oldhmInfo;
                    using (new LockAndElevate(hmLock))
                    {
                        hmInfo = new Dictionary<Address, bool[]>(1000);
                        oldhmInfo = hmInfo;
                    }
                    doIPMCNewView(gaddr, v);
                    foreach (KeyValuePair<Address, bool[]> kvp in oldhmInfo)
                    {
                        if (kvp.Key.Equals(gaddr))
                            continue;
                        Group g = Group.Lookup(kvp.Key);
                        if (g == null)
                            g = TrackingProxyLookup(kvp.Key);
                        if (g != null && g.HasFirstView)
                            doIPMCNewView(kvp.Key, g.theView);
                    }
                }
                else
                    doIPMCNewView(gaddr, v);
        }

        // This housekeeping method updates the skiplist: true if I have any members down the link
        // from me to the guy at offset 2^k to my right, for k=0...log2(N), false if not
        internal static void doIPMCNewView(Address gaddr, View v)
        {
            int myBase = v.GetMyRank(), N = v.members.Length;
            if (myBase == -1)
            {
                using (new LockAndElevate(hmLock))
                    if (hmInfo.ContainsKey(gaddr))
                        hmInfo.Remove(gaddr);
                return;
            }
            bool[] hasMembers = new bool[log2(N) + 1];
            foreach (Address m in v.members)
                if (!m.isMyAddress())
                    for (int rankOf = 0; rankOf < N; rankOf++)
                        if (m.Equals(v.members[(rankOf + myBase) % N]))
                        {
                            hasMembers[log2(rankOf)] = true;
                            break;
                        }
            using (new LockAndElevate(hmLock))
            {
                // Safely delete the old map if it had one, then remember this new mapping
                if (hmInfo.ContainsKey(gaddr))
                    hmInfo.Remove(gaddr);
                hmInfo.Add(gaddr, hasMembers);
            }
            if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                Isis.WriteLine("IPMC New View<" + gaddr + ">: members=" + Address.VectorToString(v.members) + ": new tunnel state = " + TunnelState());
        }

        internal void becomeClientOf(Address who)
        {
            if (Isis.ClientOf == null && Isis.ORACLE.theView.GetMyRank() != -1)
                // The multicast reaches all system members, but (obviously) ORACLE members ignore it
                return;
            if (Isis.ClientOf != null && Isis.ClientOf.Equals(who) == false)
                using (new LockAndElevate(Isis.RIPLock))
                    if (!Isis.RIPList.Contains(Isis.ClientOf))
                        Isis.RIPList.Add(Isis.ClientOf);
            Isis.ClientOf = who;
            Isis.OracleFailedAt = 0;
            Isis.ORACLE.theView.members[0] = who;
            Group g;
            if ((g = Group.Lookup(Isis.ORACLE.gname)) != null)
                g.GroupClose();
            ILock.Barrier(ILock.LLWAIT, ILock.LCLIENTOF).BarrierRelease(1);
        }

        private void sortThenDeliverInOrder()
        {
            Address[] Senders;
            int[] vids, msgids;
            GenerateOrdering(out Senders, out vids, out msgids, true);
        }

        private void GenerateOrdering(out Address[] senders, out int[] vids, out int[] msgids, bool flushing)
        {
            using (new LockAndElevate(OutOfOrderQueueLock))
            {
                if ((IsisSystem.Debug & IsisSystem.ORDEREDSEND) != 0)
                    Isis.WriteLine("Isis.ORDEREDSEND: GenerateOrdering, flushing=" + flushing);
                if (flushing)
                {
                    if ((IsisSystem.Debug & IsisSystem.ORDEREDSEND) != 0)
                    {
                        int cnt = 0;
                        foreach (Msg m in OutOfOrderQueue)
                            if (m.ordered)
                                ++cnt;
                        if (desiredOrderQueue.Count() != 0 || cnt > 0)
                        {
                            string doq = " ", moq = " ";
                            foreach (svi sv in desiredOrderQueue)
                                doq += sv.sender + "::" + sv.vid + ":" + sv.msgid + " ";
                            foreach (Msg m in OutOfOrderQueue)
                                moq += m.sender + "::" + m.vid + ":" + m.msgid + (m.ordered ? "*ordered" : "") + " ";
                            Isis.WriteLine("GenerateOrdering: During flush found DesiredOrderQueue = {" + doq + "}, OutOfOrderQueue = { " + moq + "}");
                        }
                    }
                    desiredOrderQueue = new List<svi>();
                    OutOfOrderQueue.Sort();
                }
                int idx = 0;
                foreach (Msg m in OutOfOrderQueue)
                {
                    if (!flushing && m.ordered)
                        continue;
                    ++idx;
                }
                senders = new Address[idx];
                vids = new int[idx];
                msgids = new int[idx];
                idx = 0;
                foreach (Msg m in OutOfOrderQueue)
                {
                    if (!flushing && m.ordered)
                        continue;
                    m.ordered = true;
                    senders[idx] = m.sender;
                    vids[idx] = m.vid;
                    msgids[idx] = m.msgid;
                    ++idx;
                }
            }
            if (flushing)
            {
                if ((IsisSystem.Debug & IsisSystem.ORDEREDSEND) != 0)
                    Isis.WriteLine("SortThenDeliverInOrder<" + gname + ">: " + OrderToString(senders, vids, msgids));
                DeliverInOrder("Isis.SortThenDeliverInOrder", senders, vids, msgids);
                desiredOrderQueue = new List<svi>();
                using (new LockAndElevate(OutOfOrderQueueLock))
                    OutOfOrderQueue = new List<Msg>();
            }
            else if ((IsisSystem.Debug & IsisSystem.ORDEREDSEND) != 0)
                Isis.WriteLine("GenerateOrdering<" + gname + ">: " + OrderToString(senders, vids, msgids));
        }

        internal bool onDOQ(Msg m)
        {
            using (new LockAndElevate(OutOfOrderQueueLock))
            {
                foreach (svi sviQE in desiredOrderQueue)
                    if (sviQE.vid == m.vid && sviQE.msgid == m.msgid && sviQE.sender.Equals(m.sender))
                        return true;
                return false;
            }
        }

        internal void DeliverInOrder(string cfrom, Address[] senders, int[] vids, int[] msgids)
        {
            int vid = -1;
            using (new LockAndElevate(ViewLock))
                if (theView != null)
                    vid = theView.viewid;
            using (new LockAndElevate(OutOfOrderQueueLock))
            {
                for (int n = 0; n < senders.Length; n++)
                {
                    if (vids[n] < vid)
                        continue;
                    foreach (svi sviQE in desiredOrderQueue)
                        if (sviQE.vid == vids[n] && sviQE.msgid == msgids[n] && sviQE.sender.Equals(senders[n]))
                            throw new IsisException("OrderedSend: multiple orderings for " + senders[n] + "::" + vids[n] + ":" + msgids[n]);
                    desiredOrderQueue.Add(new svi(senders[n], vids[n], msgids[n]));
                }
                List<Msg> toDeliver = new List<Msg>();
                svi sv;
                while ((sv = desiredOrderQueue.FirstOrDefault()) != null)
                {
                    if (sv.vid < vid)
                    {
                        desiredOrderQueue.RemoveAt(0);
                        continue;
                    }
                    bool fnd = false;
                    List<Msg> newOOQ = new List<Msg>();
                    foreach (Msg m in OutOfOrderQueue)
                        if (m.vid == sv.vid && m.msgid == sv.msgid && m.sender.Equals(sv.sender))
                        {
                            desiredOrderQueue.RemoveAt(0);
                            if (fnd)
                                throw new IsisException("DeliverInOrder<" + cfrom + ">: Duplicate redelivery of " + m.sender + "::" + m.vid + ":" + m.msgid);
                            fnd = true;
                            m.type = Msg.REDELIVERY;
                            toDeliver.Add(m);
                        }
                        else
                            newOOQ.Add(m);
                    OutOfOrderQueue = newOOQ;
                    if (!fnd)
                        break;
                }
                if (toDeliver.Count() > 0)
                    incomingSends.putFront(toDeliver);
            }
        }

        internal void GotLastSeqns(int vid, int[] LastSeqns)
        {
            if (this == Isis.ORACLE || PendingQueue == null || (flags & G_ISLARGE) != 0)
                // Before first view is known, or in ORACLE (which has a special structure), or in large groups, which implement view synchrony in a different way
                return;
            using (new LockAndElevate(PendingQueueLock))
            {
                if (theView.viewid != vid)
                    throw new IsisException("GotLastSeqns: theView.vid=" + theView.viewid + ", but GotLastSeqns has vid=" + vid);
                int myRank = theView.GetMyRank();
                if (myRank == -1) return;
                theView.LastMsg = LastSeqns;
                for (int i = 0; i < LastSeqns.Length; i++)
                    if (i == myRank || LastSeqns[i] == theView.NextIncomingMsgID[1 + i])
                        PendingQueue[1 + i] = null;
                if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                    using (new LockAndElevate(ReliableSender.ackInfoLock))
                        ReliableSender.ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: GotLastSeqns: nullify PendingQueue for <" + gname + ">r\n");
            }
        }

        internal bool IAmLeader()
        {
            if (HasFirstView == false || theView.hasFailed == null)
                return false;
            return theView.IAmLeader();
        }

        internal bool IAmRank0()
        {
            return theView.IAmRank0();
        }

        internal int nLive()
        {
            int n = 0;
            foreach (bool b in theView.hasFailed)
                if (b == false)
                    ++n;
            return n;
        }

        /// <exclude>
        /// <summary>
        /// Internal, used to ensure that lists of Groups can be sorted
        /// </summary>
        /// <returns>A hashcode for this group</returns>
        /// </exclude>
        public override int GetHashCode()
        {
            return gaddr.GetHashCode();
        }

        /// <summary>
        /// Looks up a group by Address
        /// </summary>
        /// <param name="gaddr">An Address for the desired group</param>
        /// <returns>a Group object, or null if not found</returns>
        public static Group Lookup(Address gaddr)
        {
            if (gaddr == null || Isis.ORACLE == null || Isis.ISISMEMBERS == null || gaddr.Equals(Isis.ORACLE.gaddr) || gaddr.Equals(Isis.ISISMEMBERS.gaddr))
                return null;
            Group g;
            using (new LockAndElevate(IsisGroupsLock))
                if (IsisGroups.TryGetValue(gaddr, out g))
                    return g;
            return null;
        }

        internal static Group Lookup(int vaddr)
        {
            using (new LockAndElevate(IsisGroupsLock))
                foreach (var kvp in IsisGroups)
                    if (kvp.Value.myVirtIPAddr == vaddr)
                        return kvp.Value;
            return null;
        }

        internal static Group doLookup(Address gaddr)
        {
            if (gaddr == null)
                return null;
            Group g;
            using (new LockAndElevate(IsisGroupsLock))
                if (IsisGroups.TryGetValue(gaddr, out g))
                    return g;
            if (Isis.ORACLE != null && gaddr.Equals(Isis.ORACLE.gaddr))
                return Isis.ORACLE;
            if (Isis.ISISMEMBERS != null && gaddr.Equals(Isis.ISISMEMBERS.gaddr))
                return Isis.ISISMEMBERS;
            return null;
        }

        /// <summary>
        /// Looks up a group by name
        /// </summary>
        /// <param name="gname">the group name</param>
        /// <returns>a Group object, or null if not found</returns>
        public static Group Lookup(string gname)
        {
            if (gname.Equals("ORACLE") || gname.Equals("ISISMEMBERS"))
                return null;
            using (new LockAndElevate(IsisGroupsLock))
                foreach (var kvp in IsisGroups)
                    if (kvp.Value.gname.Equals(gname))
                        return kvp.Value;
            return null;
        }

        internal static Group doLookup(string gname)
        {
            using (new LockAndElevate(IsisGroupsLock))
                foreach (var kvp in IsisGroups)
                    if (kvp.Value.gname.Equals(gname))
                        return kvp.Value;
            if (gname.Equals("ORACLE"))
                return Isis.ORACLE;
            return null;
        }

        /// <summary>
        /// Gets the current View of a group.  
        /// </summary>
        /// <returns>the current view of the group</returns>
        /// <remarks>
        /// The View is a central property of the Isis model.  Learn more about it by reading about
        /// <it>virtual synchrony</it>.  With the View you can build very sophisticated workload sharing and fault-tolerance mechanisms
        /// easily and with confidence in their correctness.
        /// </remarks>
        public View GetView() { return theView; }

        /// <summary>
        /// Gets the name of a group
        /// </summary>
        /// <returns>the group name</returns>
        public string GetName() { return gname; }

        private string PFlags(int flags)
        {
            if (flags == 0)
                return "";
            string s = "";
            for (int n = 0; (1 << n) <= flags; n++)
                if ((flags & (1 << n)) != 0)
                    s += flagNames[n] + " ";
            return "{ " + s + "}";
        }

        /// <exclude>
        /// <summary>
        /// Internal, pretty-prints the state of a group for debugging purposes
        /// </summary>
        /// <returns>string representing the group state</returns>
        /// </exclude>
        public override string ToString()
        {
            try
            {
                if (this == Isis.ORACLE && Isis.ClientOf != null)
                    return "ORACLE requests: I access the ORACLE remotely as a client of " + Isis.ClientOf;
                tokenInfo theToken;
                View theView;
                int msgid;
                using (new LockAndElevate(TokenLock))
                using (new LockAndElevate(ViewLock))
                {
                    theToken = this.theToken;
                    theView = this.theView;
                    msgid = nextMsgid;
                }
                int f;
                string stabilityInfo;
                using (new LockAndElevate(GroupFlagsLock))
                {
                    f = flags;
                    stabilityInfo = ", " + (SendStabilityNeeded ? "Need" : "Don't need") + " to send stability info (last sent at " + Isis.TimeToString(SentStableAt) + "; MaxBacklogSent=" + MaxBacklogSent + ")";
                }
                string repAddr = "[VirtIP: " + MCMDSocket.PMCAddr(myVirtIPAddr) + ", PhysIP: " + MCMDSocket.PMCAddr(myPhysIPAddr) + "]";
                string s = !HasFirstView ? "" : ("\r\n    " + theView.ToString());
                s = "group <" + gname + (where == null ? "" : "(created by " + where + ")") + ">... gaddr " + gaddr + PFlags(f) +
                    ", IP address " + repAddr + s + "\r\n    My recieved multicast count " + rcvdMcastsCnt + ", rate " + rcvdMcastsRate + ", next outgoing msgid " + msgid +
                    stabilityInfo + ", minStable=" + getMinStable();
                if ((flags & G_ISLARGE) != 0 && !isTrackingProxy)
                {
                    if (theToken != null)
                        s += "\r\n      " + theToken.ToString();
                    else
                        s += "\r\n      .... Token still uninitialized";
                    s += "\r\n      Large-group garbage collection has collected messages with (vid:id) in range (*:[0-" + gcollectedTo + "])";
                }
                using (new LockAndElevate(groupLock))
                {
                    using (new LockAndElevate(curMsgListLock))
                        foreach (KeyValuePair<Thread, Msg> kvp in curMsgList)
                            s += "\r\n  Thread " + (kvp.Key.Name == null ? "(unnamed)" : kvp.Key.Name) + " is currently busy delivering " + kvp.Value;
                    if (incomingSends != null)
                        using (new LockAndElevate(incomingSends.Lock))
                            if (incomingSends.fullSlots > 0)
                            {
                                s += "\r\n      A user-level delay during multicast delivery has allowed multicasts to enqueue:";
                                for (int i = 0; i < incomingSends.fullSlots; i++)
                                {
                                    object o = incomingSends.theBuffer[(incomingSends.gNext + i) % incomingSends.size];
                                    if (o == null)
                                        s += "\r\n        (null entry: EOF)";
                                    else if (o.GetType().Equals(typeof(Msg)))
                                        s += "\r\n        Incoming multicast ready for delivery: " + (Msg)o;
                                    else if (o.GetType().Equals(typeof(View)))
                                        s += "\r\n        " + (View)o;
                                }
                            }
                    if (incomingP2P != null)
                        using (new LockAndElevate(incomingP2P.Lock))
                            if (incomingP2P.fullSlots > 0)
                            {
                                s += "\r\n      A user-level delay during p2p delivery has allowed p2p messages to enqueue:";
                                for (int i = 0; i < incomingP2P.fullSlots; i++)
                                {
                                    object o = incomingP2P.theBuffer[(incomingSends.gNext + i) % incomingSends.size];
                                    if (o == null)
                                        s += "\r\n        (null entry: EOF)";
                                    else if (o.GetType().Equals(typeof(Msg)))
                                        s += "\r\n        Incoming P2P ready for delivery: " + (Msg)o;
                                    else if (o.GetType().Equals(typeof(View)))
                                        s += "\r\n        " + (View)o;

                                }
                            }
                }
                using (new LockAndElevate(ToDoLock))
                {
                    if (ToDo.Count() > 0)
                    {
                        foreach (Msg m in ToDo)
                            s += "\r\n      ToDo: " + m;
                    }
                }
                if (PendingQueue != null)
                    using (new LockAndElevate(PendingQueueLock))
                    {
                        for (int i = 0; i < PendingQueue.Length; i++)
                        {
                            if (PendingQueue[i] == null || PendingQueue[i].Count() == 0)
                                continue;
                            s += "\r\n         PendingQueue";
                            if ((flags & G_ISLARGE) == 0)
                            {
                                if (i > 0)
                                    s += " for sender " + theView.members[i - 1];
                                else
                                    s += " from ORACLE";
                            }
                            s += ":";
                            foreach (KeyValuePair<long, Msg> kvp in PendingQueue[i])
                                s += "\r\n            " + ((Msg)kvp.Value);
                        }
                    }
                using (new LockAndElevate(CausalOrderListLock))
                    if (CausalOrderList.Count() > 0)
                    {
                        s += "\r\n      CausalSend multicasts awaiting delivery (myVT = " + VTtoString(theView.myVT) + "):";
                        foreach (ctuple ct in CausalOrderList)
                            s += "\r\n        " + VTtoString(ct.theVT) + " " + ct.theMsg;
                    }

                using (new LockAndElevate(OutOfOrderQueueLock))
                    if (OutOfOrderQueue.Count() > 0)
                    {
                        s += "\r\n      Known ordering information: { ";
                        foreach (svi sv in desiredOrderQueue)
                            s += "[" + sv.sender + "::" + sv.vid + ":" + sv.msgid + "] ";
                        s += "}\r\n      Totally Ordered Sends Awaiting Order Info:";
                        foreach (Msg m in OutOfOrderQueue)
                            s += "\r\n        " + m;
                    }

                using (new LockAndElevate(OrderedSubsetListLock))
                    if (OrderedSubsetPQ.Count() > 0)
                    {
                        s += "\r\n      Subset multicasts awaiting order information (myTS=" + myTS + "):";
                        foreach (var kvp in OrderedSubsetPQ)
                            s += "\r\n             " + kvp;
                    }

                using (new LockAndElevate(UnstableLock))
                    if (Unstable != null && Unstable.Count() > 0)
                    {
                        s += "\r\n      Unstable multicast messages: {";
                        foreach (Msg m in Unstable)
                            s += " " + m.sender + "[" + m.vid + ":" + m.msgid + "]";
                        s += " }";
                    }
                return s;
            }
            catch (Exception e)
            {
                return "Unable to print state of group " + (gname ?? "(name unknown)") + " because " + e;
            }
        }

        /// <exclude>
        /// <summary>
        /// Internal, pretty-prints the state of a group for debugging purposes
        /// </summary>
        /// <returns>string representing the group state</returns>
        /// </exclude>
        public static string GetState()
        {
            string state = "";
            List<Group> isClone = new List<Group>();
            using (new LockAndElevate(IsisGroupsLock))
                foreach (var kvp in IsisGroups)
                    isClone.Add(kvp.Value);
            if (isClone.Count() != 0)
            {
                state += "GROUPS:\r\n";
                foreach (Group g in isClone)
                    if (g != Isis.ORACLE)
                        try
                        {
                            state += ListGroup(g) + "   ----------------------------------------------------------------------------------------------------\r\n";
                        }
                        catch (Exception)
                        {
                            if (g.gname != null)
                                state += "Isis threw an exception while trying to print the state of group <" + g.gname + ">";
                        }
            }
            state += "MEMBERSHIP ORACLE:\r\n" + (Isis.ORACLE == null ? "NO ORACLE" : ListGroup(Isis.ORACLE));
            using (new LockAndElevate(TPGroupsLock))
            {
                if (TPGroups.Count() != 0)
                {
                    state += "    TRACKING PROXIES:\r\n";
                    foreach (var kvp in TPGroups)
                    {
                        Group tpg = kvp.Value;
                        try
                        {
                            state += "       " + Group.TPtoString(tpg) + "\r\n";
                        }
                        catch (Exception)
                        {
                            if (tpg.gname != null) state += "Isis threw an exception while trying to print the state of group <" + tpg.gname + ">";
                        }
                    }
                }
            }
            if (Isis.Proposed != null && Isis.Proposed.Count() > 0)
            {
                state += "    PROPOSED VIEW DELTAS:\r\n";
                foreach (Isis.ViewDelta vd in Isis.Proposed)
                    state += "        " + vd + "\r\n";
            }
            state += vgGetState() + "\r\n";
            state += dumpStash();
            return state;
        }

        private static string ListGroup(Group g)
        {
            string s = "   " + (g.GroupOpen ? "" : "** CLOSED **") + g + "\r\n";
            if ((g.flags & G_ISLARGE) != 0)
                s += g.LGRelayGetState();
            else
                s += g.AggState();
            s += g.GetLockState();
            if ((g.flags & G_USESOOB) != 0)
                s += OOBState(g);
            return s;
        }

        /// <summary>
        /// Associate a name with a group.
        /// </summary>
        /// <param name="gname">the group name</param>
        /// <remarks>
        /// Rarely used, this method associates a group name with a group.  For a durable group the name is typically a file name 
        /// in the global file system and will be the file in which group keys and checkpoint data is saved by Isis.  Access to the
        /// file plays a central role in the Isis protection scheme: users who can't access that file can't access the group key
        /// and hence can't see the data if the group is a secured one.  In fact Isis has no admission control policies of its own:
        /// it just rides along on the file system security architecture in this sense.
        /// </remarks>
        public void Bind(String gname)
        {
            if (GroupOpen == true)
                return;
            this.gname = gname;
        }

        internal void SetMap(string where, int[] mm)
        {
            myVirtIPAddr = mm[MCMDSocket.VIRTUAL];
            myPhysIPAddr = mm[MCMDSocket.PHYSICAL];
            MCMDSocket.SetMap(where, gname, false, mm);
        }

        internal void NewView(View v, string calledFrom, int[] curMap)
        {
            View nv = null;
            NewView(v, calledFrom, curMap, ref nv);
        }

        internal void NewView(View v, string calledFrom, int[] curMap, ref View nv)
        {
            bool hadFirstView = HasFirstView;
            if ((IsisSystem.Debug & (IsisSystem.MSGIDS | IsisSystem.VIEWCHANGE | IsisSystem.STARTSEQ)) != 0)
                Isis.WriteLine("ENTERING NEWVIEW[caller:" + calledFrom + "]: Group " + gname + ",  with new view=" + v);
            if (v.viewid > 0 && v.joiners.Contains(v.members[0]))
                // This can happen if a group loses its (only) current member just as someone is joining...
                InitializeGroup(v);
            using (new LockAndElevate(GroupFlagsLock))
            {
                if ((flags & G_NEEDSTATEXFER) != 0)
                    xferWait.Release();
                flags &= ~G_NEEDSTATEXFER;
            }
            if (isTrackingProxy)
            {
                // These are used only by the ORACLE to track groups on behalf of their members
                // and by clients to track the ORACLE itself
                using (new LockAndElevate(ViewLock))
                {
                    theView = v;
                    nextMsgid = 0;
                }
                if (!hadFirstView && v.joiners.Length == 0)
                    v.joiners = new Address[] { Isis.my_address };
                HasFirstView = true;
                replayStash(this);
                List<byte[]> ae = IPMCArrivedEarly;
                List<ReliableSender.MReplayMe> mae = MsgArrivedEarly;
                IPMCArrivedEarly = null;
                MsgArrivedEarly = null;
                MCMDSocket.drainIPMCArrivedEarly(ae);
                drainEarlyArrivalMsgQ(mae);
                if (v.members.Length == 0)
                    using (new LockAndElevate(TPGroupsLock))
                        TPGroups.Remove(this.gaddr);
                return;
            }
            if (HasFirstView == false)
            {
                if (gname.Equals("ORACLE") == false && v.GetMyRank() == -1)
                {
                    Isis.WriteLine("NewView: I'm not actually IN the first view for " + gname + "\r\nThe view is " + v);
                    return;
                }
                using (new LockAndElevate(ToDoLock))
                    if (ToDo.Count() > 0)
                    {
                        List<Msg> newToDo = new List<Msg>();
                        foreach (Msg m in ToDo)
                            if (m.vid >= v.viewid)
                                newToDo.Add(m);
                        ToDo = newToDo;
                    }
            }
            if ((flags & G_ISLARGE) == 0 || !HasFirstView)
                using (new LockAndElevate(ViewLock))
                    nextMsgid = 0;
            if (curMap != null)
            {
                if (this != Isis.ORACLE || Isis.ClientOf == null)
                    SetMap("Newview<" + gname + ">", curMap);
                ReliableSender.StartGroupReader(this);
            }
            opqDrain();
            if (v.joiners.Length > 0)
            {
                foreach (Address who in v.joiners)
                    WatchEvent(who, W_JOIN);
            }
            if (v.leavers.Length > 0)
            {
                foreach (Address who in v.leavers)
                    WatchEvent(who, W_LEAVE);
            }
            if (theView != null && theView.viewid >= v.viewid)
            {
                if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                    Isis.WriteLine("NEWVIEW: IGNORE view update for Group " + gname + ", " + v);
                return;
            }
            if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                Isis.WriteLine("NEWVIEW(" + calledFrom + "): Group " + gname + ", " + v);
            SortedList<long, Msg>[] oldPendingQueue = PendingQueue;
            int PQlen;
            if ((flags & G_ISLARGE) == 0)
            {
                PQlen = v.members.Length + 1;
                // In small groups, we flush when changing views, hence Unstable can be discarded.  In large groups, the rule is different
                using (new LockAndElevate(UnstableLock))
                    Unstable = new List<Msg>();
            }
            else
                PQlen = 2;
            if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                using (new LockAndElevate(ReliableSender.ackInfoLock))
                    ReliableSender.ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: newview: reinitialize PendingQueue for <" + gname + ">r\n");
            using (new LockAndElevate(PendingQueueLock))
            {
                // Synchronized with GotAMsg, which is a real-time procedure that needs to be rather nimble...
                using (new LockAndElevate(ViewLock))
                {
                    sortThenDeliverInOrder();
                    finalizePendingSafeSends(v);
                    if (v.joiners.Length > 0)
                    {
                        if (theChkptChoser == null && v.IAmLeader())
                            v.theChkptMakers = theChkptMakers;
                        else if (theChkptChoser != null)
                            foreach (Address who in v.joiners)
                                if (theChkptChoser(v, who))
                                {
                                    v.theChkptMakers = theChkptMakers;
                                    break;
                                }
                    }
                    theView = v;
                    nRaw = 0;
                    for (int i = 0; i < myDHTnShards; i++)
                        lastVersionId[i] = -1;
                    nv = v;
                    incomingSends.put(v);
                    SortedList<long, Msg>[] oldPQ = PendingQueue;
                    PendingQueue = new SortedList<long, Msg>[PQlen];
                    for (int i = 0; i < PQlen; i++)
                        PendingQueue[i] = new SortedList<long, Msg>();
                    if (oldPQ != null && (IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                        foreach (SortedList<long, Msg> sl in oldPQ)
                            if (sl != null && sl.Count() > 0)
                            {
                                string ms = " ";
                                foreach (KeyValuePair<long, Msg> kvp in sl)
                                    ms += kvp.Value.sender + "::" + kvp.Value.vid + ":" + kvp.Value.msgid + " ";
                                Isis.WriteLine("WARNING: Newview<" + gname + ">: When installing view " + v.viewid + " found and discarded undelivered retained messages {" + ms + "}");
                            }
                }
            }
            using (new LockAndElevate(CausalOrderListLock))
                CausalOrderList = new List<ctuple>();
            HasFirstView = true;
            if ((flags & G_ISLARGE) == 0)
                resetSGAggregations();
            if (oldPendingQueue != null)
                new Thread(delegate()
                {
                    try
                    {
                        Thread.CurrentThread.Name = "replay OldPendingQueue contents in g.NewView()";
                        for (int oldi = 0; oldi < oldPendingQueue.Length && (GroupOpen || !WasOpen); oldi++)
                            if (oldPendingQueue[oldi] != null)
                                foreach (KeyValuePair<long, Msg> kvp in oldPendingQueue[oldi])
                                    if (kvp.Value.vid >= v.viewid)
                                        GotAMsg(kvp.Value, Msg.FIFOCAST, "Newview");
                                    else
                                    {
                                        if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                                            Isis.WriteLine("Discarding a duplicate in NewView: message had a stale viewid");
                                        using (new LockAndElevate(IsisSystem.RTS.Lock))
                                            ++IsisSystem.RTS.Discarded;
                                    }
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    IsisSystem.ThreadTerminationMagic();
                }).Start();
            if (durabilityMethod != null)
            {
                durabilityMethod.NewView(v);
                if (v.GetMyRank() == 0 && !IAmSafeSendLeader)
                    durabilityMethod.PlayBack();
            }
            replayStash(this);
            theView.isFinal = false;
            GroupOpen = true;
            ReliableSender.resetTheToken(this);
            UpdateShortCuts(v);

            if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.VIEWWAIT)) != 0)
                Isis.WriteLine("Releasing LLINITV barrier lock for gaddr " + gaddr + "(Lock [" + ILock.LLINITV + "][" + ILock.GetLockId(ILock.LLINITV, gaddr.GetHashCode()) + "])");
            ILock.Barrier(ILock.LLINITV, gaddr).BarrierReleaseAll();
            if (v.members.Length == 0)
                GroupClose();
        }

        /* Reset aggregation info in a small group */
        private void resetSGAggregations()
        {
            if (!HasFirstView)
            {
                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("Reinitialize the AggList<" + gname + "> called but HasFirstView=false");
                return;
            }
            View theView;
            using (new LockAndElevate(ViewLock))
                theView = this.theView;
            int N = theView.members.Length;
            if (N == 0)
                return;
            LinkedList<object>[] oldAggList;
            using (new LockAndElevate(AggListLock))
            {
                oldAggList = AggList;
                int nlevels = log2RU(N) + 1;
                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("Reinitialize the AggList<" + gname + "> for VID=" + theView.viewid + "[nlevels=" + nlevels + "] from the AggTypes List in <" + gname + ">... (it lists " + AggTypes.Count() + " types)");
                AggList = new LinkedList<object>[nlevels];
                binfo.resetBarrierList();
                for (int n = 0; n < nlevels; n++)
                {
                    AggList[n] = new LinkedList<object>();
                    foreach (AggInfo ag in AggTypes)
                    {
                        if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                            Isis.WriteLine("Calling constructor in <" + gname + "> to allocate a new aggregator of type " + ag.KVT);
                        // These are actually "aggregator" objects of some derived type
                        AggList[n].AddLast((object)(ag.myFactory.Invoke(new Object[] { this, theView.viewid, n, ag.theDel, ag.theTimeout })));
                    }
                }
            }
            if (oldAggList != null)
                for (int n = 0; n < oldAggList.Length; n++)
                    foreach (IAggregateEventHandler ae in oldAggList[n])
                        ae.AggEvent(Group.BreakWaits, theView.viewid, null, null, 0);
        }

        // Takes advantage of the fact that commit has the effect of also making anything in the SS list atomic
        // The list is sorted in a standard order and now all members of View v will also have the same list contents
        // So, we can safely deliver these messages prior to installing the new view.
        private void finalizePendingSafeSends(View v)
        {
            SortedList<SUTW, Msg> newSSList = new SortedList<SUTW, Msg>();
            using (new LockAndElevate(SSLock))
            {
                foreach (KeyValuePair<SUTW, Msg> ss in SSList)
                    if (!ss.Key.commitFlag)
                    {
                        ss.Key.commitFlag = true;
                        // The value picked must be larger than anything on the queue but also deterministic: all group members must use the same value for the same message.
                        // We also prefer to respect the ordering on message id's.  This particular TS should do the trick: these messages will move to the end of the queue
                        // and then be delivered in msgid order, ties broken by sender id
                        ss.Key.Who = ss.Key.Sender;
                        ss.Key.TS = Int32.MaxValue - 10000 + ss.Value.msgid;
                        if ((IsisSystem.Debug & IsisSystem.SAFESEND) != 0)
                            Isis.WriteLine("SAFEDELIVER[FINALIZE]: " + ss.Value.sender + "::" + ss.Value.vid + ":" + ss.Value.msgid + "... SET ORDER: " + ss.Key.Sender + "::*+" + (ss.Key.TS - (Int32.MaxValue - 10000)));
                        newSSList.Add(ss.Key, ss.Value);
                    }
                    else
                        newSSList.Add(ss.Key, ss.Value);
                SSList = newSSList;
                if ((IsisSystem.Debug & IsisSystem.SAFESEND) != 0)
                {
                    Isis.WriteLine("SAFESEND[FINALIZE]: Queue after completion and resorting: ");
                    foreach (KeyValuePair<SUTW, Msg> ss in SSList)
                        Isis.WriteLine("      (UID=" + ss.Key.Sender + "::" + ss.Key.Uid + "): " + ss.Value.sender + "::" + ss.Value.vid + ":" + ss.Value.msgid + "... ORDER: " + ss.Key.Who + "::" +
                            (ss.Key.TS > 10000 ? ss.Key.TS - (Int32.MaxValue - 10000) : ss.Key.TS) + ")");
                }
                deliverSSItems();
            }
        }

        private void drainEarlyArrivalMsgQ(List<ReliableSender.MReplayMe> mae)
        {
            if (mae == null || mae.Count() == 0)
                return;
            new Thread(delegate()
            {
                try
                {
                    Thread.CurrentThread.Name = "MsgDrain for <" + gname + ">";
                    Thread.CurrentThread.Priority = ThreadPriority.AboveNormal;
                    foreach (ReliableSender.MReplayMe m in mae)
                        ReliableSender.GotIncoming(m.type, m.gaddr, m.sender, m.minStable, m.msg, false);
                }
                catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                IsisSystem.ThreadTerminationMagic();
            }).Start();
        }

        /// <summary>
        /// Event code designating that the watched process has joined the group
        /// </summary>
        public const int W_JOIN = 0;
        /// <summary>
        /// Event code designating that the watched process has left the group
        /// </summary>
        public const int W_LEAVE = 1;

        internal void WatchEvent(Address who, int ev)
        {
            myWatches mwl = Watch[who];
            if (mwl == null)
                return;
            foreach (Watcher w in mwl.hList)
                w(ev);
        }

        /// <summary>
        /// Register a handler for incoming multicasts or queries in the group.
        /// </summary>
        /// <param name="request">The id for the requests corresponding to this handler</param>
        /// <param name="del">A request handler; it will be called for messages with matching id and type signatures</param>
        /// <remarks>
        /// A process setting up a group uses Register to register a handler for a specific request identifier, which will be a small integer
        /// counting up from 0.  The delegate is of any type you wish, and there can be multiple handlers for the same request type.  When Isis
        /// receives a message in a group it will invoke all handlers for which types exactly match.  Unfortunately, Isis can't do sophisticated
        /// type inference at this time, so it expects exact matches: if a class Dog extends Animal, and a handler defines some argument of type
        /// Animal, that handler won't be invoked for an incoming message containing a Dog, even though a Dog is an Animal.  Sorry!
        /// </remarks>
        public void RegisterHandler(int request, Delegate del)
        {
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            _RegisterHandler(request, del);
        }

        internal class ChkptMkr
        {
            Group myGroup;
            public ChkptMkr(Group g) { myGroup = g; }
            public static ChkptMkr operator +(ChkptMkr a, Delegate maker) { a.myGroup.RegisterMakeChkpt(maker); return a; }
        }

        internal ChkptMkr MakeChkpt;

        /// <summary>
        /// A process setting up a group uses RegisterMakeChkpt to register a method that will create a checkpoint
        /// for storage into a file (when terminating the group) or for state-transfer to a new joining member.
        /// </summary>
        /// <param name="maker">checkpoint making procedure, of type ChkptMaker</param>
        public void RegisterMakeChkpt(Delegate maker)
        {
            theChkptMakers.Add((ChkptMaker)maker);
        }

        /// <summary>
        /// This API is used in situations where the checkpoint for a joining member must be from a source
        /// that depends on who the joiner is.  It cannot be used by the application if the group is using the
        /// the Isis DHT, which has its own special choser, because only one choser function can be specified per group
        /// </summary>
        /// <param name="choser">Called once per joining member in all group members who were in the prior view, returns true in the single member who will 
        /// make the checkpoint on behalf of that joining member.  Notice that if many members all join at once, different sources can send the checkpoint
        /// for different receivers.  We highly recommend aiming for that kind of parallelism if you will have large numbers of joiners.  The DHT
        /// uses this scheme because for any given joiner, only certain group members are suitable checkpoint senders</param>
        /// <remarks>Using this API, a group designer can control which group member is selected to send state transfers, on a joiner by joiner basis.
        /// The method will be invoked in parallel at all group members as a new view is about to be installed, and each returns true or false.  The intent is
        /// that just one returns true for any particular joiner; it will create a checkpoint, which will be sent to that joiner.
        /// 
        /// For example, in a group that internally replicates data in multiple patterns (perhaps, it shards a relational database, with each shard
        /// on a different subset of the group members), when a new member joins a shard, the state transfer should be for that shard.  In such cases
        /// you can use a CheckpointChoser function to select the lowest ranked member in the same shard as the joiner.  If you selected some random
        /// member, or always transferred state from the rank-0 member (the leader) of the group, as in the default behavior for Isis, the sender
        /// wouldn't have that shard and hence couldn't send the needed state.
        /// 
        /// Clearly, this entire approach makes sense only if you can define a selector function that operates purely on the member's Address and the
        /// membership of the group.  For example, you might take the hashcode of an address and compute that value modulo some integer K to map the 
        /// members into the range 0..K-1.  These K "subgroups" could then play the role of shard groups.  Isis can support fancier mappings that might
        /// depend on other information, but you would need to be sure that all members have the identical state when the choser is invoked.
        /// 
        /// WARNING: If every member returns false, the joiner hangs waiting for state transfer, then a timeout for the join occurs, and the new member will be
        /// dropped from the system (it will throw a "poison" exception).  Conversely, if the member that offers to do the state transfer crashes, the join
        /// will fail, throwing a "join failed" exception.
        /// </remarks>
        public void RegisterChkptChoser(ChkptChoser choser)
        {
            if (theChkptChoser != null && theChkptChoser != choser)
                throw new IsisException("RegisterChkptChoser: Attempt to register two checkpoint chosers for group <" + gname + ">");
            theChkptChoser = (ChkptChoser)choser;
        }

        internal class ChkptLdr
        {
            Group myGroup;
            public ChkptLdr(Group g) { myGroup = g; }
            public static ChkptLdr operator +(ChkptLdr a, Delegate loader) { a.myGroup.RegisterLoadChkpt(loader); return a; }
        }

        internal ChkptLdr LoadChkpt;

        /// <summary>
        /// Register a checkpoint loading method
        /// </summary>
        /// <param name="loader">Checkpoint loading method</param>
        /// <remarks>        
        /// A process setting up a group uses RegisterLoadChkpt to define a method that will load a checkpoint as part of
        /// a state-transfer to initialize joining members if the group already is active when they join
        /// </remarks>
        public void RegisterLoadChkpt(Delegate loader)
        {
            RegisterHandler(Isis.STATEXFER, loader);
            flags |= G_NEEDSTATEXFER;
        }

        private static int tsl;

        internal bool CheckPointFileExists()
        {
            return File.Exists(myCheckpointFile + ".chkpt");
        }

        internal void LoadCheckpointFromFile()
        {
            for (int retry = 0; retry < 3; retry++)
                try
                {
                    byte[] lb = new byte[4];
                    myChkptStream = new FileStream(myCheckpointFile + ".chkpt", FileMode.Open);
                    while (myChkptStream.Read(lb, 0, 4) == 4)
                    {
                        int len = ((int)lb[3] << 24) | ((int)lb[2] << 16) | ((int)lb[1] << 8) | (int)lb[0];
                        if (len < 0 || len > Isis.ISIS_MAXMSGLEN + 1024)
                            throw new IsisException("Group <" + gname + ">: corrupted checkpoint file[len=" + len + " must be within 0.." + (Isis.ISIS_MAXMSGLEN + 1024) + "]");
                        byte[] buffer = new byte[len];
                        if (myChkptStream.Read(buffer, 0, len) != len)
                            throw new IOException("unexpected EOF");
                        if (!Msg.VerifySignature(buffer, 0, len))
                            throw new IsisException("Group <" + gname + ">: corrupted checkpoint file[signature verification failed]");
                        if (userSpecifiedKey)
                            decipherBuf(buffer);
                        object[] obs = Msg.BArrayToObjects(buffer);
                        object[] args = new object[obs.Length + 1];
                        args[0] = Isis.STATEXFER;
                        for (int i = 0; i < obs.Length; i++)
                            args[i + 1] = obs[i];
                        cbAction(null, -1, -1, Isis.my_address, args);
                    }
                    myChkptStream.Close();
                    myChkptStream = null;
                    return;
                }
                catch
                {
                }
            throw new IsisException("LoadCheckPointFromFile: I/O exception");
        }

        // Old style
        internal void doRegister(int request, Delegate del)
        {
            _RegisterHandler(request, del);
        }

        private void _RegisterHandler(int request, Delegate del)
        {
            Handlers[request] += new CallBack(false, del);
        }

        Random AesSeed = null;

        /// <summary>
        /// Places a group into secure mode, selecting a new randomly generated AES key for the initial create, and later reading the
        /// key from the checkpoint .hdr file when a new member joins.
        /// </summary>
        /// <remarks>In this mode, Isis2 stores the key in plain text form in the .hdr file, and checkpoints are also
        /// unencrypted.  Thus anyone with file system permissions could gain access to the group data.  Use the overload
        /// of SetSecure lat lets you specify a key that you manage the key outside of Isis2 if you want to avoid this particular threat. </remarks>
        public void SetSecure()
        {
            if (GroupOpen)
                throw new IsisException("Illegal to call SetSecure after Group.Join");
            if (myCheckpointFile == null)
                throw new IsisException("Setsecure: To have Isis2 pick a group key, must first call g.Persistent()");
            flags |= G_SECURE;
            InitializeMyAes();
            myAESkey = new byte[myAes.KeySize >> 3];
            AesSeed.NextBytes(myAESkey);
            SetAesKey(myAes, myAESkey);
        }

        /// <summary>
        /// Places a group into secure mode using a specified key that the end-user must manage externally to Isis2.  
        /// </summary>
        /// <param name="theKey">myAesKeySize-byte AES key</param>
        /// <remarks>The given key must be the same one used by other group members or some form of horrible problem will occur, probably a crash.
        /// Also, keep in mind that an attacker with root privilages could still use a debugger to attach to the program and read myAESKey from memory.
        /// Still, this form of security is stronger than if you let Isis2 pick its own keys.</remarks>
        public void SetSecure(byte[] theKey)
        {
            flags |= G_SECURE;
            InitializeMyAes();
            if ((theKey.Length << 3) != myAes.KeySize)
                throw new ArgumentException("Key must be a byte[" + (myAes.KeySize >> 3) + "] vector");
            userSpecifiedKey = true;
            myAESkey = theKey;
            if (myAESkey == null || (flags & G_SECURE) == 0)
            {
                Isis.WriteLine("Group <" + gname + ">, set myAes=null in SetAEsKey");
                myAes = null;
                return;
            }
            else
                SetAesKey(myAes, myAESkey);
        }

        internal void InitializeMyAes()
        {
            if (AesSeed != null || (flags & G_SECURE) == 0)
                return;
            AesSeed = new System.Random((int)DateTime.Now.Ticks);
            doInitializeAes(out myAes);
        }

        internal static void doInitializeAes(out Aes theAes)
        {
            theAes = Aes.Create();
            if (theAes == null)
                throw new IsisException("myAes null after Aes.Create()");
            theAes.Padding = PaddingMode.None;
            KeySizes[] ks = theAes.LegalKeySizes;
            int ksbits = 0;
            int bsbits = 0;
            foreach (KeySizes k in ks)
                ksbits = Math.Max(ksbits, k.MaxSize);
            ks = theAes.LegalBlockSizes;
            foreach (KeySizes k in ks)
                bsbits = Math.Max(bsbits, k.MaxSize);
            theAes.KeySize = ksbits;
            theAes.BlockSize = bsbits;
        }

        internal static void SetAesKey(Aes theAes, byte[] theAesKey)
        {
            bool allZero = true;
            int nb = (theAes.KeySize >> 3);
            for (int b = 0; b < nb; b++)
                if (theAesKey[b] != 0)
                    allZero = false;
            if (allZero)
            {
                theAesKey = null;
                return;
            }
            theAes.Key = theAesKey;
        }

        /// <summary>
        /// Declares that this group is persistent, gives a file in which state should be stored.  Isis will append the needed
        /// file name extensions, using .ckpt for the checkpoint and .hdr for a header containing various system data.  A file
        /// named filename.bak contains the previous checkpoint and is available to the user in case a checkpoint becomes corrupted
        /// </summary>
        /// <param name="filename">A file name, normally for sharing between all the group members.</param>
        public void Persistent(string filename)
        {
            myCheckpointFile = filename;
        }

        internal bool isPersistent()
        {
            return myCheckpointFile != null;
        }

        internal void setupPersistentFile()
        {
            int retryCnt = 0;
        tryAgain:
            byte[] theKey;
            if (tsl == 0)
                tsl = Msg.toBArray((long)0).Length;
            try
            {
                if ((flags & G_SECURE) != 0)
                {
                    InitializeMyAes();
                    theKey = new byte[myAes.KeySize >> 3];
                }
                else
                    theKey = new byte[16];
                for (int retry = 0; retry < 15; retry++)
                {
                    try
                    {
                        myChkptStream = new FileStream(myCheckpointFile + ".hdr", FileMode.Open);
                        if (myChkptStream.Read(theKey, 0, theKey.Length) == theKey.Length)
                            break;
                        myChkptStream.Close();
                        myChkptStream = null;
                        Thread.Sleep(500 * (Isis.my_address.GetHashCode() % 10 + retry));
                    }
                    catch (FileNotFoundException)
                    {
                        // Lucky me: I get to create the persistent file
                        throw;
                    }
                    catch (IOException)
                    {
                        // Can happen if a race arises with many trying to join concurrently
                        Thread.Sleep(500 * (Isis.my_address.GetHashCode() % 10 + retry));
                    }
                }
                if (myChkptStream == null)
                    throw new IsisException("Unable to open checkpoint file" + myCheckpointFile + ".hdr (tried 15 times)");
                if ((flags & G_SECURE) != 0 && !userSpecifiedKey)
                {
                    myAESkey = theKey;
                    SetAesKey(myAes, myAESkey);
                }
                long ts = TypeSignature(this);
                byte[] tsb = new byte[Msg.toBArray(ts).Length];
                if (myChkptStream.Read(tsb, 0, tsl) != tsl)
                    throw new IOException("Unexpected EOF");
                long chkts = (long)Msg.BArrayToObjects(tsb)[0];
                if (ts != chkts)
                    Isis.WriteLine("WARNING: Checkpoint for group <" + gname + "> records a different set of handlers/type signatures than this member");
                myChkptStream.Close();
                myChkptStream = null;
            }
            catch (FileNotFoundException)
            {
                // Ignore; this arises normally during the initial group create
                try
                {
                    myChkptStream = new FileStream(myCheckpointFile + ".hdr", FileMode.Create);
                }
                catch
                {
                    if (++retryCnt == 5)
                        throw new IsisException("Unable to create or access " + myCheckpointFile + ".hdr");
                    Thread.Sleep(500 * (Isis.my_address.GetHashCode() % 10 + retryCnt));
                    goto tryAgain;
                }
                if ((flags & G_SECURE) != 0)
                {
                    theKey = myAESkey;
                    if (theKey == null)
                        theKey = new byte[myAes.KeySize >> 3];
                }
                else
                    theKey = new byte[16];
                byte[] kOut = new byte[theKey.Length];
                if ((flags & G_SECURE) != 0 && !userSpecifiedKey)
                    Array.Copy(theKey, kOut, theKey.Length);
                myChkptStream.Write(kOut, 0, theKey.Length);
                long ts = TypeSignature(this);
                byte[] tsb = Msg.toBArray(ts);
                myChkptStream.Write(tsb, 0, tsb.Length);
                myChkptStream.Close();
                myChkptStream = null;
                return;
            }
            catch (Exception e)
            {
                throw new IsisException("Unexpected file I/O exception " + e + " on checkpoint file");
            }
        }

        internal bool makingCheckpoint = false;

        /// <summary>
        /// Causes the current group state to be written to the checkpoint file
        /// </summary>
        public void MakeCheckpoint(View v)
        {
            if (!IsisCallStart())
                return;
            makingCheckpoint = true;
            myChkptStream = new FileStream(myCheckpointFile + ".tmp", FileMode.Create);
            inhibitEOC = true;
            if (theChkptMakers.Count() > 0)
                foreach (ChkptMaker cpm in theChkptMakers)
                    cpm(v);
            inhibitEOC = false;
            if (makingCheckpoint)
                SendChkpt();
            IsisCallDone();
        }

        internal void MakeCheckpointIfLeader()
        {
            if (theView.IAmLeader())
                MakeCheckpoint(theView);
            if (CheckpointFrequency > 0)
                Isis.OnTimerThread(CheckpointFrequency, MakeCheckpointIfLeader);
        }

        /// <summary>
        /// Completion identifier used in DiskLogger
        /// </summary>
        public class CompletionTag : IComparable, IEqualityComparer
        {
            /// <summary>
            /// Sender of the SafeSend
            /// </summary>
            public Address sender;
            /// <summary>
            /// ViewID in which it was sent
            /// </summary>
            public int vid;
            /// <summary>
            /// Msgid of the message
            /// </summary>
            public int msgid;

            internal bool ordered;      // True if final ordering is known
            internal bool done;         // True if application has done its update
            internal Msg theMsg;

            /// <ignore></ignore>
            public CompletionTag(Msg m)
            {
                if (m.sender == null || m.sender.isNull() || m.vid == Msg.UNINITIALIZED || m.msgid == Msg.UNINITIALIZED)
                    throw new IsisException("CompletionTag: illegal msg sender/vid/msgid");
                sender = m.sender;
                vid = m.vid;
                msgid = m.msgid;
                theMsg = m;
            }

            /// <ignore></ignore>
            public int CompareTo(CompletionTag other)
            {
                if (!sender.Equals(other.sender))
                    return sender.GetHashCode() - other.GetHashCode();
                if (vid != other.vid)
                    return vid - other.vid;
                return msgid - other.msgid;
            }

            /// <ignore></ignore>
            public int CompareTo(object other)
            {
                return CompareTo((CompletionTag)other);
            }

            /// <ignore></ignore>
            public override bool Equals(object obj)
            {
                if (obj == null || !obj.GetType().Equals(typeof(CompletionTag)))
                    return false;
                return CompareTo((CompletionTag)obj) == 0;
            }

            /// <ignore></ignore>
            public new bool Equals(object o1, object o2)
            {

                if (o1 == null || !o1.GetType().Equals(typeof(CompletionTag)))
                    return false;
                if (o2 == null || !o2.GetType().Equals(typeof(CompletionTag)))
                    return false;
                return ((CompletionTag)o1).CompareTo((CompletionTag)o2) == 0;
            }

            /// <ignore></ignore>
            public override int GetHashCode()
            {
                return sender.GetHashCode() + vid * 100010057 + msgid * 71043311;
            }

            /// <ignore></ignore>
            public int GetHashCode(object other)
            {
                return other.GetHashCode();
            }

            /// <ignore></ignore>
            public override string ToString()
            {
                return "[" + sender.ToString() + "::" + vid + ":" + msgid + "]";
            }
        }

        /// <summary>
        /// Durability Methods must implement this API
        /// </summary>
        public interface IDurability
        {
            /// <summary>
            /// Logs a message, returns a new CompletionTag object for it
            /// </summary>
            /// <param name="m">The message to log</param>
            /// <returns>The associated sender:vid:msgid</returns>
            CompletionTag LogMsg(Msg m);
            /// <summary>
            /// Returns the CompletionTag of the most recently logged message
            /// </summary>
            /// <returns>The associated sender:vid:msgid</returns>
            CompletionTag GetCompletionTag();
            /// <summary>
            /// Called when the SafeSend sets the deliver order for a pending message
            /// </summary>
            /// <param name="m">The associated message</param>
            void SetOrder(Msg m);
            /// <summary>
            /// Starts an asynchronous update, later user should call Done(ct)
            /// </summary>
            /// <param name="ct">The associated completion tag</param>
            void BeginAsyncUpdate(CompletionTag ct);
            /// <summary>
            /// Called by the application when done processing an update
            /// </summary>
            /// <param name="ct">The associated completion tag</param>
            void Done(CompletionTag ct);
            /// <summary>
            /// Invoked by Isis when a new view becomes defined.  Will be called first, they PlayBack(), in the leader.
            /// </summary>
            /// <param name="v"></param>
            void NewView(View v);
            /// <summary>
            /// On reinitialization of a group after all members fail, or when the view changes
            /// and a new process assumes the role of group leader, invoked by Isis2 (in the leader only) to "play back"
            /// any messages that might not be stable in the group.  Ideally, the new leader would
            /// only need to take actions after a recovery from a full group failure, at which point
            /// it should replay all messages that didn't reach the state of being "Done" at all the
            /// members where they were delivered during the previous run.  It isn't possible to avoid
            /// duplicates so that becomes the problem of the application, which will need to filter them
            /// out.  
            /// </summary>
            void PlayBack();
            /// <summary>
            /// Tells the logger to shut itself down, called on group Terminate()
            /// </summary>
            void Shutdown();
        }

        internal IDurability durabilityMethod;
        volatile internal bool IAmSafeSendLeader;

        /// <summary>
        /// Overrides the default (in-memory) durability rule for SafeSend
        /// </summary>
        /// <param name="theMethod">An object implementing the IDurability API</param>
        public void SetDurabilityMethod(IDurability theMethod)
        {
            if (this.durabilityMethod != null)
                throw new IsisException("SetDurabilityMethod: method was already registered");
            this.durabilityMethod = theMethod;
        }

        internal delegate void DLQuery(Address[] senders, int[] vids, int[] msgids);
        internal delegate void DLDone(int nCleanup, Address[] senders, int[] vids, int[] msgids);
        internal delegate void DLCleanup();

        /// <summary>
        /// This build-in class provides log-file based durability for SafeSend.  Requires a prior call to g.SetSafeSendThreshold with a small integer constant
        /// specifying how many of the group members must maintain logs.  Throws SafeSendException if an attempt is made to issue a SafeSend in a view with fewer than
        /// this number of members.
        /// </summary>
        public class DiskLogger : IDurability, IDisposable
        {
            internal bool disposed = false;
            internal bool dirty = false;
            internal LockObject theLock;
            internal Group theGroup;
            internal Thread theThread;
            internal string theFileName = "no name";
            internal FileStream theFileStream;
            internal int myRank = -1;
            internal Semaphore SleepOn = new Semaphore(0, int.MaxValue);
            List<CompletionTag> PendingList = new List<CompletionTag>();
            List<CompletionTag> AsyncList = new List<CompletionTag>();
            List<CompletionTag> DoneList = new List<CompletionTag>();

            /// <ignore></ignore>
            public DiskLogger(Group g, string filename)
            {
                if (g.safeSendThreshold <= 1)
                    throw new SafeSendException("DiskLogger requires SafeSendThreshold >= 2");
                theGroup = g;
                theFileName = filename;
                theLock = new LockObject("DiskLogger<" + theGroup.gname + ">");
                doSetup();
            }

            void IDisposable.Dispose()
            {
                Dispose(true);
            }

            internal void Dispose(bool disposing)
            {
                if (disposing)
                {
                    using (new LockAndElevate(theLock))
                        if (disposed)
                            return;
                        else
                            disposed = true;
                    SleepOn.Release(1);
                    SleepOn.Dispose();
                    SleepOn = null;
                }
            }

            /// <ignore></ignore>
            public CompletionTag LogMsg(Msg m)
            {
                using (new LockAndElevate(theLock))
                {
                    dirty = true;
                    return LogMsg(theFileStream, m);
                }
            }

            // Must call with appropriate lock
            private CompletionTag LogMsg(FileStream whichFileStream, Msg m)
            {
                if (myRank == -1 || myRank >= theGroup.GetSafeSendThreshold())
                    return null;
                CompletionTag ct = new CompletionTag(m);
                using (new LockAndElevate(theLock))
                {
                    if (whichFileStream == null)
                        throw new SafeSendException("Error: attempted to do a SafeSend in a group that has fewer than SafeSendThreshold members");
                    if (whichFileStream == theFileStream)
                    {
                        foreach (CompletionTag oct in PendingList)
                            if (oct.sender.Equals(ct.sender) && oct.vid == ct.vid && oct.msgid == ct.msgid)
                                return ct;
                        PendingList.Add(ct);
                    }
                    WriteObjects(m);
                }
                return ct;
            }

            internal void WriteObjects(params object[] obs)
            {
                byte[] ba = Msg.toBArray(obs);
                byte[] baLen = new byte[4];
                int bl = ba.Length;
                baLen[0] = (byte)((bl >> 24) & 0xFF);
                baLen[1] = (byte)((bl >> 16) & 0xFF);
                baLen[2] = (byte)((bl >> 8) & 0xFF);
                baLen[3] = (byte)(bl & 0xFF);
                theFileStream.Seek(0, SeekOrigin.End);
                theFileStream.Write(baLen, 0, 4);
                theFileStream.Write(ba, 0, ba.Length);
                theFileStream.Flush(true);
            }

            /// <ignore></ignore>
            public CompletionTag GetCompletionTag()
            {
                Msg m = theGroup.curMsg();
                if (m == null)
                    throw new IsisException("DiskLogger: curmsg null");
                return new CompletionTag(m);
            }

            /// <ignore></ignore>
            /// Called when the order is determined for message m, and these calls match the sequential delivery order
            public void SetOrder(Msg m)
            {
                if (myRank == -1 || myRank >= theGroup.GetSafeSendThreshold())
                    return;
                bool fnd = false;
                CompletionTag mct = m.ct;
                if (mct == null)
                    throw new IsisException("ct null in SetOrder");
                using (new LockAndElevate(theLock))
                {
                    List<CompletionTag> newPendingList = new List<CompletionTag>();
                    foreach (CompletionTag ct in PendingList)
                    {
                        if (ct.sender.Equals(mct.sender) && ct.vid == mct.vid && ct.msgid == mct.msgid)
                        {
                            fnd = true;
                            if (ct.ordered)
                                return;
                            else
                                continue;
                        }
                        if (ct.ordered)
                        {
                            newPendingList.Add(ct);
                            continue;
                        }
                        if (m != null)
                        {
                            m.ct.ordered = true;
                            newPendingList.Add(m.ct);
                            m = null;
                        }
                        newPendingList.Add(ct);
                    }
                    if (m != null)
                    {
                        m.ct.ordered = true;
                        newPendingList.Add(m.ct);
                        m = null;
                    }
                    if (!fnd)
                        throw new IsisException("DiskLogger.SetOrder: couldn't find " + mct.sender + "::" + mct.vid + ":" + mct.msgid);
                    PendingList = newPendingList;
                }
            }

            /// <ignore></ignore>
            public void BeginAsyncUpdate(CompletionTag ct)
            {
                using (new LockAndElevate(theLock))
                    AsyncList.Add(ct);
            }

            /// <ignore></ignore>
            public void Done(CompletionTag ct)
            {
                using (new LockAndElevate(theLock))
                    if (AsyncList.Contains(ct))
                    {
                        // In this case, expects to be called twice: once being the automated event from the callback handler, once by the asynchronous completion logic
                        AsyncList.Remove(ct);
                    }
                    else if (!DoneList.Contains(ct))
                    {
                        DoneList.Add(ct);
                        dirty = true;
                        if (theGroup.IAmSafeSendLeader && DoneList.Count() == 1000)
                            SleepOn.Release(1);
                    }
            }

            /// <ignore></ignore>
            public void Shutdown()
            {
                SleepOn.Release(1);
            }

            /// <ignore></ignore>
            public void NewView(View v)
            {
                using (new LockAndElevate(theLock))
                {
                    int rank = v.GetMyRank();
                    if (rank == -1 || rank >= theGroup.GetSafeSendThreshold())
                        return;
                    if (myRank != rank && theFileStream != null)
                    {
                        myRank = rank;
                        theFileStream.Flush(true);
                        theFileStream.Close();
#if !__MonoCS__
                        theFileStream.Dispose();
#endif // !__MonoCS__
                        Thread.Sleep(50);
                        theFileStream = null;
                        if (rank == -1)
                            return;
                    }
                    else if (theFileStream != null)
                        return;
                    else
                        myRank = rank;
                    if (theFileStream == null)
                    {
                        for (int retry = 0; retry < 60; retry++)
                        {
                            try
                            {
                                // Open it in exclusive mode to be sure that the previous owner, if any, has finished writing to it.  Could require a few tries
                                theFileStream = new FileStream(theFileName + "-" + myRank + ".dat", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
                                break;
                            }
                            catch (IOException e)
                            {
                                if ((IsisSystem.Debug & IsisSystem.DISKLOGGER) != 0)
                                    Isis.WriteLine("Attempt failed, IOException=" + e);
                                Thread.Sleep(500);
                            }
                        }
                        if (theFileStream == null)
                            throw new IsisException("DiskLogger: can't access " + theFileName + "-" + myRank + ".dat (tried for 30 secs)");
                        ReadLog();
                    }
                    if (rank == 0 && theThread == null)
                    {
                        theThread = new Thread(delegate()
                        {
                            try
                            {
                                Thread.CurrentThread.Name = "DiskLogger<" + theGroup.gname + ">: garbage collector";
                                Timeout myTO = new Timeout(Isis.ISIS_DEFAULTTIMEOUT, Timeout.TO_NULLREPLY);
                                int myThreshold = theGroup.GetSafeSendThreshold() - 1;
                                SleepOn.WaitOne(30 * 1000);
                                while (IsisSystem.IsisActive && (theGroup.GroupOpen || !theGroup.WasOpen))
                                {
                                    Address[] senders;
                                    int[] vids;
                                    int[] msgids;
                                    List<bool[]> status = new List<bool[]>();
                                    // First, construct a list of completion tags that should be checked, in delivery order
                                    using (new LockAndElevate(theLock))
                                    {
                                        if (!dirty)
                                            continue;
                                        senders = new Address[DoneList.Count()];
                                        vids = new int[DoneList.Count()];
                                        msgids = new int[DoneList.Count()];
                                        int idx = 0;
                                        foreach (CompletionTag ct in DoneList)
                                        {
                                            senders[idx] = ct.sender;
                                            vids[idx] = ct.vid;
                                            msgids[idx] = ct.msgid;
                                            ++idx;
                                        }
                                    }
                                    // Now find out which are done everywhere; question is where is the "cutoff point", again in delivery order
                                    if (theGroup.Query(myThreshold, myTO, Isis.DISKLOGGER, senders, vids, msgids, new EOLMarker(), status) == myThreshold)
                                    {
                                        int idx = 0, ndx = 0;
                                        while (idx < senders.Length)
                                        {
                                            bool allTrue = true;
                                            foreach (bool[] ba in status)
                                                if (!ba[idx])
                                                {
                                                    allTrue = false;
                                                    break;
                                                }
                                            if (allTrue)
                                            {
                                                senders[ndx] = senders[idx];
                                                vids[ndx] = vids[idx];
                                                msgids[ndx] = msgids[idx];
                                                ++ndx;
                                            }
                                            ++idx;
                                        }
                                        if (ndx > 0)
                                        {
                                            Isis.ArrayResize<Address>(ref senders, ndx);
                                            Isis.ArrayResize<int>(ref vids, ndx);
                                            Isis.ArrayResize<int>(ref msgids, ndx);
                                            using (new LockAndElevate(theLock))
                                                SetDoneBit(senders, vids, msgids);
                                            WriteObjects(ndx, senders, vids, msgids);
                                            List<string> oks = new List<string>();
                                            // Finally, if everyone successfully logs that those are collectable, safe to collect. 
                                            if (theGroup.Query(myThreshold, myTO, Isis.DISKLOGGER, ndx, senders, vids, msgids, new EOLMarker(), oks) == myThreshold)
                                                theGroup.Query(myThreshold, myTO, Isis.DISKLOGGER, new EOLMarker(), new List<string>());
                                        }
                                    }
                                    SleepOn.WaitOne(30 * 1000);
                                }
                            }
                            catch (IsisShutdown) { return; }
                            catch (ThreadInterruptedException) { return; }
                            IsisSystem.ThreadTerminationMagic();
                        });
                        theThread.Start();
                    }
                }
            }

            private void doSetup()
            {
                // Called as the first phase of garbage collection to gather statistics on the specified completion tags
                theGroup.doRegister(Isis.DISKLOGGER, (DLQuery)delegate(Address[] senders, int[] vids, int[] msgids)
                {
                    if (myRank <= 0 || myRank >= theGroup.GetSafeSendThreshold())
                    {
                        theGroup.NullReply();
                        return;
                    }
                    string inq = " ";
                    for (int i = 0; i < senders.Length; i++)
                        inq += senders[i] + "::" + vids[i] + ":" + msgids[i] + " ";
                    bool[] myStatus = new bool[senders.Length];
                    using (new LockAndElevate(theLock))
                    {
                        foreach (CompletionTag ct in PendingList)
                            for (int idx = 0; idx < senders.Length; idx++)
                                if (ct.done && senders[idx].Equals(ct.sender) && vids[idx] == ct.vid && msgids[idx] == ct.msgid)
                                {
                                    myStatus[idx] = true;
                                    break;
                                }
                        foreach (CompletionTag ct in DoneList)
                            for (int idx = 0; idx < senders.Length; idx++)
                                if (senders[idx].Equals(ct.sender) && vids[idx] == ct.vid && msgids[idx] == ct.msgid)
                                {
                                    myStatus[idx] = true;
                                    break;
                                }
                    }
                    string ms = "";
                    foreach (bool b in myStatus)
                        ms += b ? "1 " : "0 ";
                    if ((IsisSystem.Debug & IsisSystem.DISKLOGGER) != 0)
                        Isis.WriteLine("Disklogger: Responding to an inquiry {" + inq + "}, myStatus={" + ms + "}");
                    theGroup.doReply(myStatus);
                });
                // Second phase notes that these message are now collectable but doesn't collect them yet
                theGroup.doRegister(Isis.DISKLOGGER, (DLDone)delegate(int howMany, Address[] senders, int[] vids, int[] msgids)
                {
                    if (myRank <= 0 || myRank >= theGroup.GetSafeSendThreshold())
                    {
                        theGroup.NullReply();
                        return;
                    }
                    string inq = " ";
                    for (int i = 0; i < senders.Length; i++)
                        inq += senders[i] + "::" + vids[i] + ":" + msgids[i] + " ";
                    using (new LockAndElevate(theLock))
                    {
                        SetDoneBit(senders, vids, msgids);
                        WriteObjects(howMany, senders, vids, msgids);
                    }
                    if ((IsisSystem.Debug & IsisSystem.DISKLOGGER) != 0)
                        Isis.WriteLine("Disklogger: Responding to an prepare message {" + inq + "}");
                    theGroup.doReply("OK");
                });
                // Now can cleanup the log, if you wish
                theGroup.doRegister(Isis.DISKLOGGER, (DLCleanup)delegate()
                {
                    if (myRank == -1 || myRank >= theGroup.GetSafeSendThreshold())
                        return;
                    if ((IsisSystem.Debug & IsisSystem.DISKLOGGER) != 0)
                        Isis.WriteLine("Disklogger: Recieved a commit message");
                    RewriteLog(false);
                    theGroup.doReply("OK");
                });
            }

            internal void RewriteLog(bool verbose)
            {
                using (new LockAndElevate(theLock))
                {
                    try
                    {
                        int nc = 0;
                        if (verbose)
                            Isis.WriteLine("... creating temp file <" + theFileName + "-" + myRank + ".tmp>");
                        FileStream newFileStream = new FileStream(theFileName + "-" + myRank + ".tmp", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
                        List<CompletionTag> newPendingList = new List<CompletionTag>();
                        foreach (CompletionTag ct in PendingList)
                            if (!ct.done)
                            {
                                newPendingList.Add(ct);
                                LogMsg(newFileStream, ct.theMsg);
                            }
                            else
                                ++nc;
                        if (verbose)
                            Isis.WriteLine("... Wrote " + (PendingList.Count() - nc) + " SafeSend messages to new log file");
                        PendingList = newPendingList;
                        theFileStream.Flush();
                        theFileStream.Close();
                        theFileStream.Dispose();
                        newFileStream.Flush();
                        newFileStream.Close();
                        newFileStream.Dispose();
                        Thread.Sleep(250);
                        if (verbose)
                            Isis.WriteLine("... Renaming file");
                        File.Replace(theFileName + "-" + myRank + ".tmp", theFileName + "-" + myRank + ".dat", theFileName + "-" + myRank + ".bak");
                        Thread.Sleep(250);
                        theFileStream = new FileStream(theFileName + "-" + myRank + ".dat", FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
                        if ((IsisSystem.Debug & IsisSystem.DISKLOGGER) != 0)
                            Isis.WriteLine("Disklogger: Successful garbage collection of " + nc + " SafeSend messages");
                        if (verbose)
                            Isis.WriteLine("... DiskLogger logfile repair successful");
                    }
                    catch (IOException e)
                    {
                        throw new IsisException("DiskLogger: Unable to handle I/O exception " + e);
                    }
                }
            }

            private int SetDoneBit(Address[] senders, int[] vids, int[] msgids)
            {
                int fnd = 0;
                foreach (CompletionTag ct in PendingList)
                {
                    for (int idx = 0; idx < senders.Length; idx++)
                        if (senders[idx].Equals(ct.sender) && vids[idx] == ct.vid && msgids[idx] == ct.msgid)
                        {
                            ct.done = true;
                            ++fnd;
                            break;
                        }
                }
                return fnd;
            }

            /// <ignore></ignore>
            public void PlayBack()
            {
                List<Msg> toResend = new List<Msg>();
                if ((IsisSystem.Debug & IsisSystem.DISKLOGGER) != 0)
                    Isis.WriteLine("** Disklogger: entering PlayBack");
                using (new LockAndElevate(theLock))
                {
                    if (theFileStream == null)
                        return;
                    View theView;
                    using (new LockAndElevate(theGroup.ViewLock))
                        theView = theGroup.theView;
                    if (theView.members.Length < theGroup.GetSafeSendThreshold())
                        return;
                    theGroup.IAmSafeSendLeader = true;
                    List<CompletionTag> newPendingList = new List<CompletionTag>();
                    foreach (CompletionTag ct in PendingList)
                        if (!ct.done)
                            toResend.Add(ct.theMsg);
                        else
                            newPendingList.Add(ct);
                    PendingList = newPendingList;
                }
                if (toResend.Count() > 0)
                {
                    new Thread(delegate()
                    {
                        try
                        {
                            Thread.CurrentThread.Name = "SafeSend:Playback<" + theGroup.gname + ">";
                            foreach (Msg m in toResend)
                                if (IsisSystem.IsisActive)
                                    theGroup.SafeSend(Msg.BArrayToObjects(m.payload));
                        }
                        catch (IsisShutdown)
                        {
                        }
                        IsisSystem.ThreadTerminationMagic();
                    }).Start();
                }
            }

            // Call holds theLock
            internal void ReadLog()
            {
                byte[] ba = new byte[4];
                bool corrupt = false;
                theFileStream.Seek(0, SeekOrigin.Begin);
                while (theFileStream.Read(ba, 0, 4) == 4)
                {
                    int len = (ba[0] << 24) | (ba[1] << 16) | (ba[2] << 8) | ba[3];
                    ba = new byte[Math.Abs(len)];
                    int nb;
                    if ((nb = theFileStream.Read(ba, 0, len)) == len)
                    {
                        object[] obs = Msg.BArrayToObjects(ba);
                        if (Msg.CheckTypes(obs, typeof(Msg)))
                        {
                            CompletionTag ct = new CompletionTag((Msg)Msg.BArrayToObjects(ba)[0]);
                            bool fnd = false;
                            foreach (CompletionTag oct in PendingList)
                                if (oct.sender.Equals(ct.sender) && oct.vid == ct.vid && oct.msgid == ct.msgid)
                                {
                                    fnd = true;
                                    break;
                                }
                            if (!fnd)
                                PendingList.Add(ct);
                        }
                        else if (Msg.CheckTypes(obs, typeof(int), typeof(Address[]), typeof(int[]), typeof(int[])))
                        {
                            int x = 1;      //x = 0; int howMany = (int)obs[x++];
                            Address[] senders = (Address[])obs[x++];
                            int[] vids = (int[])obs[x++];
                            int[] msgids = (int[])obs[x];
                            foreach (CompletionTag ct in PendingList)
                                for (int idx = 0; idx < senders.Length; idx++)
                                    if (senders[idx].Equals(ct.sender) && vids[idx] == ct.vid && msgids[idx] == ct.msgid)
                                    {
                                        ct.done = true;
                                        break;
                                    }
                        }
                        else
                        {
                            corrupt = true;
                            Isis.WriteLine("Warning: DiskLogger unable to interpret data read from a log file... truncating and initiating autorepair");
                            break;
                        }
                    }
                    else
                    {
                        corrupt = true;
                        Isis.WriteLine("Warning: DiskLogger expected " + len + " bytes but EOF after " + nb + " bytes.... truncating and initiating autorepair");
                        break;
                    }
                }
                if (corrupt)
                {
                    Isis.WriteLine("WARNING: Isis detected a damaged or partially written DiskLogger log file... auto-repairing");
                    RewriteLog(true);
                }
                int nd = 0;
                foreach (CompletionTag ct in PendingList)
                {
                    if (ct.done)
                        ++nd;
                }
                if ((IsisSystem.Debug & IsisSystem.DISKLOGGER) != 0)
                    Isis.WriteLine("Disklogger ReadLog finished.  Read " + PendingList.Count() + " messages, of which " + nd + " were flagged as completed");
            }
        }

        /// <summary>
        /// Requests that the state of the group be checkpointed automatically at the specified interval
        /// </summary>
        /// <param name="interval">inter-checkpoint interval in seconds</param>
        public void SetCheckpointFrequency(int interval)
        {
            CheckpointFrequency = Math.Max(-1, Math.Min(interval, int.MaxValue / 1000) * 1000);
            if (CheckpointFrequency > 0)
                Isis.OnTimerThread(CheckpointFrequency, MakeCheckpointIfLeader);
        }

        /// <summary>Register view callback handler. </summary>
        /// <param name="cbproc">Handler for view callbacks</param>
        /// <remarks>
        /// A process setting up a group calls RegisterViewCB to register a callback handler that will be invoked when a new View is received.
        /// Read about the <it>virtual synchrony model</it> to learn more about the View property of groups, which is a key feature of the Isis system
        /// and will be very useful to you in designing applications that make full use of Isis.
        /// </remarks>
        public void RegisterViewHandler(Delegate cbproc)
        {
            VHCallBack vcb = new VHCallBack(false, (ViewHandler)cbproc);
            using (new LockAndElevate(ViewHandlers.vhListLock))
                ViewHandlers.vhList.Add(vcb);
        }

        internal class Initer
        {
            Group myGroup;
            public Initer(Group g) { myGroup = g; }
            public static Initer operator +(Initer a, Initializer maker) { a.myGroup.RegisterInitializer(maker); return a; }
        }

        internal Initer Initializer;

        /// <summary>
        /// A process setting up a group calls RegisterInitializer to declare the method that will initialize the group if this member turns out to create it
        /// </summary>
        /// <param name="initproc"></param>
        public void RegisterInitializer(Initializer initproc)
        {
            theInitializer = initproc;
        }

        internal void doRegisterViewCB(VHCallBack vcb)
        {
            using (new LockAndElevate(ViewHandlers.vhListLock))
                ViewHandlers.vhList.Add(vcb);
        }

        private void TypeCheck(object[] obs)
        {
            if (obs == null || obs.Length < 1) throw new IsisException("Wrong number of arguments");
            if (obs[0].GetType().Equals(typeof(Msg))) return;
            if (obs[0].GetType().Equals(typeof(int)) == false) throw new IsisException("First argument should be a request type");

            int request = (int)obs[0];
            if (Handlers[request] == null)
                return;

            foreach (CallBack cb in Handlers[request].hList)
                if (TypeMatch(obs, cb))
                    return;

            string ts = "";
            foreach (object o in obs)
                ts += o.GetType() + "...";
            throw new IsisException("No callback for request " + Isis.rToString((int)obs[0]) + " matches the provided type signature: " + ts);
        }

        internal int rcode(Object[] obs)
        {
            object rc = obs[0];
            Type t = rc.GetType();
            if (t.Equals(typeof(int)))
                return (int)rc;
            if (t.Equals(typeof(byte)))
                return (int)(byte)rc;
            Isis.WriteLine("Rcode(obs.Length=" + obs.Length + "): obs[0]=" + rc);
            throw new IsisException("Request code was of type " + rc.GetType().ToString() + " but expected byte or int!");
        }

        private static bool TypeMatch(object[] obs, CallBack cb)
        {
            if (cb.cbProc.ptypes.Length != obs.Length - 1)
                return false;
            for (int i = 0; i < cb.cbProc.ptypes.Length; i++)
                if (obs[i + 1] == null || (!obs[i + 1].GetType().Equals(cb.cbProc.ptypes[i]) && !cb.cbProc.ptypes[i].Equals(typeof(object))))
                    return false;
            return true;
        }

        private static bool DHTTypeMatch(object[] obs, CallBack cb)
        {
            if (cb.cbProc.ptypes.Length != obs.Length + 1)
                return false;
            for (int i = 0; i < obs.Length; i++)
                if (obs[i] == null || (!obs[i].GetType().Equals(cb.cbProc.ptypes[i]) && !cb.cbProc.ptypes[i].Equals(typeof(object))))
                    return false;
            return true;
        }

        // Returns a 64-bit value computed from an MD5 hash of the type signature of the group
        // MD5 is actually a 128-bit result, so we just fold it in half and XOR the parts
        internal static long TypeSignature(Group g)
        {
            if (g.TypeSig != 0)
                return g.TypeSig;
            List<string> sigs = new List<string>();
            int ridx = 0;
            foreach (myHandlers mh in g.Handlers.ListofhLists)
                if (mh != null)
                    foreach (CallBack cb in mh.hList)
                    {
                        string s = "[" + ridx++ + "]";
                        foreach (Type pt in cb.cbProc.ptypes)
                            s += pt + ":";
                        sigs.Add(s);
                    }
            Group.tokenInfo theToken;
            using (new LockAndElevate(g.TokenLock))
                theToken = g.theToken;
            if (theToken != null)
                foreach (AggInfo ag in g.AggTypes)
                    sigs.Add(ag.KVT);
            sigs.Sort();
            string sig = "";
            foreach (string s in sigs)
                sig = sig + s + ";";
            if ((g.flags & G_SECURE) != 0)
                sig += "(secure)";
            g.TypeSigStr = sig;
            if ((IsisSystem.Debug & IsisSystem.TYPESIGS) != 0)
                Isis.WriteLine("Computing type signature for " + g.gname + ", long form is " + sig);
            using (MemoryStream ms = new MemoryStream(Msg.StringToBytes(sig)))
            {
                using (HMAC hm = new HMACMD5(new byte[8] { 56, 78, 9, 23, 10, 87, 33, 11 }))
                {
                    byte[] ba = hm.ComputeHash(ms);
                    long rval = 0;
                    for (int i = 0; i < ba.Length; i++)
                        rval ^= (((long)ba[i]) & 0xFF) << ((i & 3) << 3);
                    if ((IsisSystem.Debug & IsisSystem.TYPESIGS) != 0) Isis.WriteLine("... returning compressed version " + rval);
                    if (rval == 0)
                        rval = 0x1010101001010101;
                    return rval;
                }
            }
        }

        internal const int CREATE = 0x0001;
        internal const int JOIN = 0x0002;
        internal const int CANBEORACLE = 0x0004;

        /// <summary>
        /// A process calls Join to join or create a group; Isis will figure out what to do.  Set up the group in advance by registering data types, handlers, aggregators.
        /// </summary>
        public void Join()
        {
            Join(0L);
        }

        /// <summary>
        /// Join a group, specifying an offset into the group state
        /// </summary>
        /// <param name="off">offset into the group's state</param>
        public void Join(long off)
        {
            if (GroupOpen)
                throw new IsisException("Group Join/Create but the group was already active");
            ThrashingCheck();
            using (new LockAndElevate(UniversalP2PHandlers.uhListLock))
            using (new LockAndElevate(UniversalMHandlers.uhListLock))
                if (UniversalP2PHandlers.uhList.Count() > 0 || UniversalMHandlers.uhList.Count() > 0)
                    hasUniversalHandlers = true;
            if (myCheckpointFile != null)
                setupPersistentFile();
            Group[] groups = new Group[] { this };
            if (this != Isis.ORACLE && this != Isis.ISISMEMBERS)
                Handlers.locked = UniversalMHandlers.locked = UniversalP2PHandlers.locked = true;
            doJoin(CREATE | JOIN, 0, groups, off);
            JoinWait(groups);
        }

        /// <summary>
        /// A process calls JoinExisting to join a group that must be preexisting.  Set up the group in advance by registering data types, handlers, aggregators.
        /// </summary>
        public void JoinExisting()
        {
            JoinExisting(0L);
        }

        /// <summary>
        /// Join an existing group, specifying an offset into the group state
        /// </summary>
        /// <param name="off">offset into the group's state</param>
        public void JoinExisting(long off)
        {
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            ThrashingCheck();
            Group[] groups = new Group[] { this };
            doJoin(JOIN, 0, groups, off);
            JoinWait(groups);
        }

        /// <summary>
        /// A process calls Create to create a new group.  Set up the group in advance by registering data types, handlers, aggregators.
        /// </summary>
        public void Create()
        {
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            Group[] groups = new Group[] { this };
            doJoin(CREATE | JOIN, 0, groups, 0L);
            JoinWait(groups);
        }

        private void ThrashingCheck()
        {
            using (new LockAndElevate(RecentlyLeftLock))
                if (RecentlyLeft.Contains(gaddr))
                    // This is to prevent developers from building applications that thrash, putting Isis under heavy stress
                    throw new IsisException("Isis join error: Illegal to rejoin an existing group within 5 minutes after leaving it.");
        }

        /// <summary>
        /// A process calls multiCreate to create a set of groups as a single atomic action. 
        /// </summary>
        /// <param name="groups">List of groups to create</param>
        /// <remarks>Set up the groups in advance by registering data types, handlers, aggregators.</remarks>
        public static void multiCreate(Group[] groups)
        {
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            doJoin(CREATE | JOIN, 0, groups, 0L);
            JoinWait(groups);
        }

        /// <summary>
        /// A master process calls this overload of multiCreate to create a set of groups containing a set of members as a single atomic action. 
        /// </summary>
        /// <param name="workers">List of processes to initially include in the groups</param>
        /// <param name="groups">List of groups</param>
        /// <remarks>Set up the groups in advance by registering data types, handlers, aggregators.</remarks>
        public static void multiCreate(Address[] workers, Group[] groups)
        {
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            doJoin(CREATE | JOIN, 0, workers, groups, 0L);
        }

        /// <summary>
        /// A master process calls this overload of multiJoin to add a set of workers to a list of groups as a single atomic action.  
        /// </summary>
        /// <param name="workers">List of processes to add to the groups</param>
        /// <param name="groups">List of groups</param>
        /// <remarks>Set up the groups in advance by registering data types, handlers, aggregators.</remarks>
        public static void multiJoin(Address[] workers, Group[] groups)
        {
            multiJoin(workers, groups, 0L);
        }

        /// <summary>
        /// Joint a list of groups, specifying an offset into the group state
        /// </summary>
        /// <param name="workers">List of processes to add to the groups</param>
        /// <param name="groups">List of groups</param>
        /// <param name="offset">Offset into the group state</param>
        public static void multiJoin(Address[] workers, Group[] groups, long offset)
        {
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            foreach (Group g in groups)
                if (!g.HasFirstView)
                    throw new IsisException("multiJoin: Can only be called by a current member of <" + g.gname + ">");
            doJoin(CREATE | JOIN, 0, workers, groups, offset);
        }

        /// <summary>
        /// A process calls multiJoin to join a set of group as a single atomic action.  
        /// </summary>
        /// <param name="groups">List of groups to join</param>
        /// <remarks>Set up the groups in advance by registering data types, handlers, aggregators.</remarks>
        public static void multiJoin(Group[] groups)
        {
            multiJoin(groups, 0L);
        }

        /// <summary>
        /// Join a set of groups as a single atomic action
        /// </summary>
        /// <param name="groups">List of groups to join</param>
        /// <param name="offset">Offset into the group state</param>
        public static void multiJoin(Group[] groups, long offset)
        {
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            doJoin(CREATE | JOIN, 0, groups, offset);
            JoinWait(groups);
        }

        /// <summary>
        /// Manager for a group calls Terminate to garbage collect the group, leaving members operational.
        /// </summary>
        /// <remarks>Once this call is issued, if any member tries to issue Isis system calls on the group, those calls will
        /// fail and a warning will print to the console.  So don't call Terminate while your group is still active. </remarks>
        public void Terminate()
        {
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            if (!GroupOpen || (flags & G_TERMINATING) != 0)
                return;
            doTerminate(new Group[] { this });
        }

        /// <summary>
        /// Manager for a set of groups calls <c>multiTerminate</c> to garbage collect the groups as a single action, leaving members operational.
        /// </summary>
        /// <param name="groups">List of the groups to terminate</param>
        public static void multiTerminate(Group[] groups)
        {
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            foreach (Group g in groups)
                if (!g.HasFirstView)
                    throw new IsisException("multiTerminate: Caller must Join <" + g.gname + "> before calling multiTerminate");
            doTerminate(groups);
        }

        /// <summary>
        /// A process calls HasFailed to report a failure
        /// </summary>
        /// <param name="who">Address of the failed process</param>
        /// <remarks>
        /// The caller and failed process must both belong to the current view.  The target will be forced out of the
        /// Isis system (a poison pill will be sent to it just in case it is still running) and it will drop out of
        /// all groups to which it belongs.
        /// </remarks>
        public void HasFailed(Address who)
        {
            if (theView.GetRankOf(who) == -1)
                return;
            Isis.NodeHasFailed(who, "(From app-level HasFailed)", false);
        }

        internal Semaphore xferWait = new Semaphore(0, Int32.MaxValue);

        private static void JoinWait(Group[] groups)
        {
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            foreach (Group g in groups)
            {
                if (g.HasFirstView == false)
                {
                    if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.VIEWWAIT)) != 0)
                        Isis.WriteLine("Waiting on LLINITV barrier lock for gaddr " + g.gaddr + "(Lock [" + ILock.LLINITV + "][" + ILock.GetLockId(ILock.LLINITV, g.gaddr.GetHashCode()) + "])");
                    if (g.HasFirstView == false)
                        ILock.Barrier(ILock.LLINITV, g.gaddr).BarrierWait();
                    if ((IsisSystem.Debug & (IsisSystem.VIEWCHANGE | IsisSystem.VIEWWAIT)) != 0)
                        Isis.WriteLine("After LLINITV barrier lock for gaddr " + g.gaddr + "(Lock [" + ILock.LLINITV + "][" + ILock.GetLockId(ILock.LLINITV, g.gaddr.GetHashCode()) + "])");
                    if (g.joinFailed) // Join but it didn't exist, or Create but it did exist
                        throw new IsisException(g.reason);
                    if (g.HasFirstView == false)
                        throw new IsisException("JoinWait barrier returned but group has no view!!! " + g.gaddr);
                    if (g.theView.GetMyRank() == -1)
                        throw new IsisException("JoinWait barrier returned but I'm not in " + Address.VectorToString(g.theView.members) + " (gaddr " + g.gaddr + ")");
                }
                if ((g.flags & G_NEEDSTATEXFER) != 0)
                    g.xferWait.WaitOne();
                int lcnt = 0;
                while (!g.CallbacksDone)
                    if (lcnt++ > 250)
                        throw new IsisException("Stuck waiting for NEWVIEW callbacks to occur in <" + g.gname + ">.Join()");
                    else
                        Thread.Sleep(100);
            }
        }
        internal class vGroup
        {
            internal Address creator;
            internal Address vGroupAddr;
            internal Address[] vGMembers;

            internal vGroup(Address who, Address[] members)
            {
                creator = Isis.my_address;
                vGroupAddr = who;
                vGMembers = members;
            }

            internal vGroup(Address createdBy, Address who, Address[] members)
            {
                creator = createdBy;
                vGroupAddr = who;
                vGMembers = members;
            }
        }

        internal static List<vGroup> vGList = new List<vGroup>();
        internal static LockObject vGLock = new LockObject("vGLock");
        internal static System.Random rand = new System.Random();

        internal static Address virtualGroup(Address[] members)
        {
            using (new LockAndElevate(vGLock))
            {
                foreach (vGroup vg in vGList)
                    if (vGMatch(vg.vGMembers, members))
                        return vg.vGroupAddr;
            }
            vGroup nvg = new vGroup(newVGAddress(members), members);
            using (new LockAndElevate(vGLock))
                vGList.Add(nvg);
            return nvg.vGroupAddr;
        }

        internal static bool vGMatch(Address[] l0, Address[] l1)
        {
            if (l0.Length != l1.Length)
                return false;
            for (int i = 0; i < l0.Length; i++)
                if (!l0[i].Equals(l1[i]))
                    return false;
            return true;
        }

        internal static Address newVGAddress(Address[] members)
        {
            // Creates a placeholder Address() with a negative pid (used to notice that this is a placeholder), picked
            // to reduce risk of collisions if we test Isis with multiple "masters" on one machine.  Otherwise the home
            // IP address should suffice to avoid problems.
            Address nVGA = new Address(Isis.my_address.home, -rand.Next(1, int.MaxValue));
            if (Isis.ClientOf != null)
                Isis.ORACLE.P2PSend(Isis.ClientOf, Isis.RELAYREGISTERVG, Isis.my_address, nVGA, members);
            else
                Isis.ORACLE.doSend(false, false, Isis.REGISTERVG, Isis.my_address, nVGA, members);
            return nVGA;
        }

        internal static void noteVGMap(Address master, Address nVGA, Address[] members)
        {
            if (master.isMyAddress() || Isis.ISISMEMBERS.theView.GetRankOf(master) == -1)
                return;
            vGroup nvg = new vGroup(newVGAddress(members), members);
            using (new LockAndElevate(vGLock))
            {
                foreach (vGroup vg in vGList)
                    if (vg.vGroupAddr.Equals(nVGA))
                        throw new IsisException("Reuse of nVGA " + nVGA);
                vGList.Add(nvg);
            }
            Isis.ISISMEMBERS.Watch[master] += (Watcher)delegate(int ev)
            {
                using (new LockAndElevate(vGLock))
                    vGList.Remove(nvg);
            };
        }

        internal static vGroup vGLookup(Address vga)
        {
            using (new LockAndElevate(vGLock))
                foreach (vGroup vg in vGList)
                    if (vg.vGroupAddr.Equals(vga))
                        return vg;
            return null;
        }

        internal static string vgGetState()
        {
            string s = "MEMBER-SET SHORTCUTS: ";
            using (new LockAndElevate(vGLock))
                if (vGList.Count() == 0)
                    return "";
                else foreach (vGroup vg in vGList)
                        s += "\r\n  Creator: " + vg.creator + ", members={" + Address.VectorToString(vg.vGMembers) + "}";
            return s;
        }

        internal static void doJoin(int mode, int level, Address[] newMembers, Group[] groups, long offset)
        {
            doJoin(mode, level, virtualGroup(newMembers), groups, offset);
        }

        internal static void doJoin(int mode, int level, Group[] groups, long offset)
        {
            doJoin(mode, level, Isis.my_address, groups, offset);
        }

        internal static void doJoin(int mode, int level, Address who, Group[] groups, long offset)
        {
            int groupsToJoin = 0;
            foreach (Group g in groups)
            {
                if (g.GroupOpen == true && who.isMyAddress())
                {
                    if ((IsisSystem.Debug & IsisSystem.GVELOGIC) != 0)
                        Isis.WriteLine("WARNING: You asked me to doJoin (mode " + mode + ") group " + g.gname + ", but it was already open");
                    continue;
                }
                g.TypeSig = TypeSignature(g);
                groupsToJoin++;
                g.GroupOpen = true;
            }

            if (groupsToJoin == 0)
                return;

            Address[] gaddrs = new Address[groups.Length];
            string[] gnames = new string[groups.Length];
            int[] flags = new int[groups.Length];
            long[] tsigs = new long[groups.Length];
            for (int idx = 0; idx < groups.Length; idx++)
            {
                gnames[idx] = groups[idx].gname;
                gaddrs[idx] = groups[idx].gaddr;
                flags[idx] = groups[idx].flags;
                tsigs[idx] = groups[idx].TypeSig;
                if (gnames[idx] == null || gaddrs[idx] == null)
                    throw new IsisException("Something is wrong in doJoin");
            }

            if (Isis.ClientOf != null)
            {
                if ((IsisSystem.Debug & IsisSystem.RELAYLOGIC) != 0)
                {
                    Isis.WriteLine("Invoking Isis.RELAYJOIN: sender " + Isis.my_address + ", client of " + Isis.ClientOf + ", vectors of length " + gnames.Length);
                    for (int n = 0; n < gnames.Length; n++)
                        Isis.WriteLine("   gname " + gnames[n] + ", gaddrs " + gaddrs[n] + ", gsigs " + tsigs[n]);
                }
                bool ISend = false;
                while (Isis.ClientOf != null && Isis.ORACLE.doP2PQuery(Isis.ClientOf, new Timeout(Isis.ISIS_DEFAULTTIMEOUT, Timeout.TO_FAILURE, "RELAYJOIN"),
                    Isis.RELAYJOIN, who, Isis.ORACLE.uids++, mode, gnames, gaddrs, offset, tsigs, flags).Length == 0)
                {
                    if (Isis.newClientOfCnt++ == 0)
                        ISend = true;
                    Isis.ClientOf = null;
                    if (ISend)
                        Isis.ORACLE.doSend(false, false, Isis.JOIN, Isis.my_address, mode, new string[] { "ORACLE" }, new Address[] { Isis.ORACLE.gaddr }, offset, new long[] { 0 }, new int[] { 0 }, ++IsisSystem.IsisJoinCounter);
                    ILock.Barrier(ILock.LLWAIT, ILock.LCLIENTOF).BarrierWait(Isis.ORACLE);
                }
            }
            else
            {
                if ((IsisSystem.Debug & IsisSystem.GVELOGIC) != 0)
                {
                    string gns = "", isls = " ";
                    foreach (string gs in gnames)
                        gns += " " + gs;
                    foreach (int f in flags)
                        isls += f + " ";
                    Isis.WriteLine("Sending Isis.JOIN requests to the ORACLE for gnames [" + gns + " ], gaddrs [" + Address.VectorToString(gaddrs) + "], isLarge{" + isls + "}");
                }
                Isis.ORACLE.doSend(false, false, Isis.JOIN, who, mode, gnames, gaddrs, offset, tsigs, flags, ++IsisSystem.IsisJoinCounter);
            }

            IsisSystem.IsisRestarting = false;
        }

        internal static void doTerminate(Group[] groups)
        {
            Address[] gaddrs = new Address[groups.Length];
            int idx = 0;
            foreach (Group g in groups)
            {
                if (g.GroupOpen)
                {
                    g.Query(ALL, new Timeout(15000, Timeout.TO_FAILURE), Isis.TERMINATE, g.gaddr, EOL);
                    g.Flush();
                    gaddrs[idx++] = g.gaddr;
                }
            }
            while (Isis.ClientOf != null)
                if (Isis.ORACLE.doP2PQuery(Isis.ClientOf, new Timeout(Isis.ISIS_DEFAULTTIMEOUT, Timeout.TO_FAILURE, "RELAYTERM"), Isis.RELAYTERM, Isis.my_address, Isis.ORACLE.uids++, gaddrs).Length != 0)
                    return;
            Isis.ORACLE.doSend(false, false, Isis.TERMINATE, Isis.my_address, Isis.ORACLE.uids++, gaddrs);
            TerminationWait(groups);
        }

        internal const int NPING = 3;

        // Every Isis member pings randomly selected members of groups to which it belongs at a rate of
        // NPINGs per second.  For example, if NPING=3, each member picks three other processes (if it can find that many)
        // and just nudges them, once per second.  This triggers various failure detection logic so that if a process
        // fails, or even an entire group fails, we'll figure it out quickly and clean up the mess
        internal static void GroupMemberHeartBeat()
        {
            while (!IsisSystem.IsisActive)
                Thread.Sleep(250);
            try
            {
                while (IsisSystem.IsisActive)
                {
                    IsisSystem.RTS.ThreadCntrs[9]++;
                    System.Random rand = new System.Random();
                    List<Address> alist = new List<Address>();
                    using (new LockAndElevate(IsisGroupsLock))
                    {
                        int N = 0;
                        foreach (var kvp in IsisGroups)
                            if (kvp.Value.HasFirstView)
                                N += kvp.Value.theView.members.Length;
                        if (N > 1)
                        {
                            for (int k = 0; alist.Count() < NPING && k < N; k++)
                            {
                                int which = rand.Next(N);
                                int cnt = 0;
                                foreach (var kvp in IsisGroups)
                                {
                                    Group g = kvp.Value;
                                    if (g.HasFirstView)
                                    {
                                        int off = which - cnt;
                                        if (off >= 0 && off < g.theView.members.Length)
                                        {
                                            if (!g.theView.members[off].isMyAddress() && !alist.Contains(g.theView.members[off]))
                                                alist.Add(g.theView.members[off]);
                                        }
                                        cnt += g.theView.members.Length;
                                    }
                                }
                            }
                        }
                    }
                    foreach (Address a in alist)
                        SendPing(a);
                    Thread.Sleep(30000);
                }
            }
            catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
            IsisSystem.ThreadTerminationMagic();
        }

        internal static MCMDSocket.GRPair[] GroupRates()
        {
            List<MCMDSocket.GRPair> GRPList = new List<MCMDSocket.GRPair>();
            using (new LockAndElevate(IsisGroupsLock))
            {
                foreach (var kvp in IsisGroups)
                {
                    Group g = kvp.Value;
                    if ((IsisSystem.Debug & IsisSystem.MCMD) != 0)
                        Isis.WriteLine("MCMD: Updating rate info for gaddr=" + g.gaddr + ", g.rcvdMcastsRate = (rate:" + g.rcvdMcastsRate + " cnt:" + g.rcvdMcastsCnt + ")/2 giving " + (g.rcvdMcastsRate + g.rcvdMcastsCnt) / 2);
                    g.rcvdMcastsRate = (g.rcvdMcastsRate + g.rcvdMcastsCnt) / 2;
                    g.rcvdMcastsCnt = 0;
                    GRPList.Add(new MCMDSocket.GRPair(g.gaddr, g.rcvdMcastsRate));
                }
            }
            return GRPList.ToArray();
        }

        internal void OracleHeartBeat()
        {
            try
            {
                int n = 1, looped = 0, counter = 0;
                List<Address> CleanUp = new List<Address>();
                while (IsisSystem.IsisActive && (GroupOpen || !WasOpen))
                {
                    IsisSystem.RTS.ThreadCntrs[10]++;
                    Thread.Sleep(1000);
                    if (HasFirstView == false)
                    {
                        if (looped++ == 5)
                        {
                            if ((IsisSystem.Debug & IsisSystem.STARTSEQ) != 0)
                                Isis.WriteLine("ISIS ORACLE SERVICE: Restarted");
                            IsisSystem.IsisRestarting = false;
                            View v = new View("ORACLE", Isis.ORACLE.gaddr, new Address[] { Isis.my_address }, 0, false);
                            Isis.ORACLE.NewView(v, "ORACLE", null);

                            ReliableSender.StartGroupReader(Isis.ORACLE);

                            // Runs only in the current leader
                            Isis.ORACLE.LeaderMode = true;
                            Isis.ORACLE.GroupOpen = true;
                            Isis.OracleViewThread = new Thread(Isis.OracleViewTask);
                            Isis.OracleViewThread.Name = "Isis <ORACLE> View Thread";
                            Isis.IAmOracle = true;
                            Isis.OracleViewThread.Start();
                            MCMDSocket.RunMappingTask();
                        }
                    }
                    else if (theView.IAmLeader())
                    {
                        n = (n + 1) % theView.members.Length;
                        if (theView.members[n].isMyAddress() == false && theView.hasFailed[n] == false)
                            SendPing(theView.members[n]);
                    }
                    else if ((n = theView.GetMyRank()) > 0)
                    {
                        while (theView.hasFailed[--n])
                            if (n == 0)
                                throw new IsisException("Total failure of the ORACLE group (of which I'm a client): Isis must shut down.");
                        SendPing(theView.members[n]);
                    }
                    if ((counter % 1) == 0)
                    {
                        // Every second, the Oracle members....
                        Address sendTo = null;
                        using (new LockAndElevate(TPGroupsLock))
                        {
                            foreach (var kvp in TPGroups)
                            {
                                Group g = kvp.Value;
                                int r = theView.GetMyRank();
                                if (HasFirstView && r != -1 && (((g.gaddr.GetHashCode() + counter) % theView.members.Length) == r))
                                    if (g.theView != null && g.theView.members.Length > 0)
                                        for (int i = 0; i < g.theView.members.Length; i++)
                                            if (!g.theView.hasFailed[i])
                                            {
                                                // ... take turns pinging the leaders of each group (the first non-failed member in the view)
                                                // The idea is to spread load but also to be sure that every group definitely does get pinged once per second
                                                sendTo = g.theView.members[i];
                                                break;
                                            }
                            }
                        }
                        if (sendTo != null)
                            SendPing(sendTo);
                    }
                    if (++counter % 600 == 0)
                    {
                        /* Every five minutes */
                        using (new LockAndElevate(Isis.RIPLock))
                        {
                            foreach (Address a in CleanUp)
                                Isis.RIPList.Remove(a);
                            CleanUp = new List<Address>();
                            foreach (Address a in Isis.RIPList)
                                CleanUp.Add(a);
                        }
                    }
                }
            }
            catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
            IsisSystem.ThreadTerminationMagic();
        }


        // Ping him, unless something is already in the send queue or the send queue length is excessive
        private static void SendPing(Address sendTo)
        {
            if (sendTo.isMyAddress())
                return;
            bool fnd = false;
            if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                using (new LockAndElevate(ReliableSender.ackInfoLock))
                    ReliableSender.ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: SendPing to " + sendTo + "\r\n");
            using (new LockAndElevate(ReliableSender.PendingSendBufferLock))
            {
                if (ReliableSender.PendingSendBuffer.Count() > Isis.ISIS_ASYNCMTOTALLIMIT * 2)
                    fnd = true;
                else foreach (ReliableSender.MsgDesc md in ReliableSender.PendingSendBuffer)
                        if (md.dest.Equals(sendTo))
                        {
                            fnd = true;
                            break;
                        }
            }
            if (!fnd)
                ReliableSender.SendP2P(Msg.ISPING, sendTo, null, new byte[0], true);
        }

        internal LockObject RateLock = new LockObject("RateLock");
        internal int accumulatedMsgTokens = 0;
        internal int accumulatedByteTokens = 0;
        internal int msgTokenRate = int.MaxValue;
        internal int byteTokenRate = int.MaxValue;
        internal int sleepDelay = 0;

        internal Thread RateThread;

        /// <summary>
        /// Sets a rate limit for multicasts expressed as messages per second
        /// </summary>
        /// <param name="msgspersec">Limit on messages per time unit</param>
        public void SetRateLimit(int msgspersec)
        {
            SetRateLimit(msgspersec, int.MaxValue, 1000);
        }

        /// <summary>
        /// Sets a rate limit for multicasts in this group.  
        /// </summary>
        /// <param name="msgspertimeunit">Limit on messages per time unit</param>
        /// <param name="bytespertimeunit">Limit on bytes per time unit</param>
        /// <param name="timeunit">Time unit for the limits in milliseconds</param>
        public void SetRateLimit(int msgspertimeunit, int bytespertimeunit, int timeunit)
        {
            using (new LockAndElevate(RateLock))
            {
                msgTokenRate = msgspertimeunit;
                byteTokenRate = bytespertimeunit;
                sleepDelay = Math.Max(timeunit, 50);
                if (RateThread == null)
                {
                    RateThread = new Thread(delegate()
                    {
                        while (!IsisSystem.IsisActive)
                            Thread.Sleep(250);
                        try
                        {
                            while (IsisSystem.IsisActive)
                            {
                                IsisSystem.RTS.ThreadCntrs[11]++;
                                Monitor.Enter(RateLock);
                                if (msgTokenRate >= int.MaxValue / 2)
                                    accumulatedMsgTokens = msgTokenRate;
                                else
                                    accumulatedMsgTokens = accumulatedMsgTokens / 2 + msgTokenRate;
                                if (byteTokenRate >= int.MaxValue / 2)
                                    accumulatedByteTokens = byteTokenRate;
                                else
                                    accumulatedByteTokens = accumulatedByteTokens / 2 + byteTokenRate;
                                Monitor.PulseAll(RateLock);
                                Monitor.Exit(RateLock);
                                Thread.Sleep(sleepDelay);
                            }
                        }
                        catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                        IsisSystem.ThreadTerminationMagic();
                    });
                    RateThread.Name = "Group [" + gname + "]: Leaky bucket rate controller thread";
                    RateThread.Start();
                }
            }
        }

        // Implements the internal logic for the leaky bucket rate controller, supported optionally on a per-group basis
        internal void RateLimit(int nb)
        {
            if (RateThread == null)
                return;
            Monitor.Enter(RateLock);
            while (accumulatedMsgTokens == 0 || accumulatedByteTokens < nb)
                Monitor.Wait(RateLock);
            accumulatedMsgTokens--;
            accumulatedByteTokens -= nb;
            Monitor.Exit(RateLock);
        }

        List<KeyValuePair<int, long>> IdsByThreadId = new List<KeyValuePair<int, long>>();
        LockObject IdsByThreadIdLock = new LockObject("IdsByThreadIdLock");

        internal long newLoggingId()
        {
            long myId;
            using (new LockAndElevate(IdsByThreadIdLock))
            {
                myId = myLoggingId++;
                foreach (KeyValuePair<int, long> kvp in IdsByThreadId)
                    if (kvp.Key == Thread.CurrentThread.ManagedThreadId)
                    {
                        IdsByThreadId.Remove(kvp);
                        break;
                    }
                IdsByThreadId.Add(new KeyValuePair<int, long>(Thread.CurrentThread.ManagedThreadId, myId));
            }
            return myId;
        }

        internal long lookupLoggingId()
        {
            using (new LockAndElevate(IdsByThreadIdLock))
                foreach (KeyValuePair<int, long> kvp in IdsByThreadId)
                    if (kvp.Key == Thread.CurrentThread.ManagedThreadId)
                        return kvp.Value;
            return -1;
        }

        internal bool usesSubsetSend;

        /// <summary>
        /// A totally ordered Isis multicast primitive.  All members receive these messages in the same view and in the identical order.
        /// </summary>
        /// <param name="obs">A variable-length argument list specifying a request ID and a set of arguments that should match some handler.</param>
        /// <remarks>
        /// A totally ordered Isis multicast primitive that can be extremely fast, particularly if sent by the group leader.  OrderedSend is a <it>non-durable multicast</it>.
        ///
        /// This non-durable ordered form of Send is often the right choice, as it offers total ordering but can be very fast, especially if sent from the group leader. 
        /// 
        /// Basically, maps to an IP multicast, but delivery is then delayed until ordering information is specified by the leader (the leader gets a special
        /// break: in that member, OrderedSend is performed using the cheapest Send because FIFO is all we need in that case).
        /// Synchronized with respect to membership changes in accordance with the virtual synchrony model, which is a strong and
        /// useful property.  The first parameter is an identifier for the message handler to invoke (a small integer) and the remaining
        /// are the arguments, which are objects of types known to Isis.  An exception will be thrown if the group doesn't have a message handler for
        /// the specified request id, and matching the number and types of objects you specified.
        ///
        /// Read more about the <it>virtual synchrony model</it> to learn about the options
        /// and how to pick the fastest mechanism for the setting in which your application will run.
        /// </remarks>
        public void OrderedSend(params object[] obs)
        {
            if (!IsisCallStart())
                return;
            if (obs.Length > 1 && obs[0].GetType().Equals(typeof(int)) && IsSubsetSend(obs[1]))
            {
                object[] newobs = new object[obs.Length + 1];
                newobs[0] = obs[1];
                for (int i = 0; i < obs.Length; i++)
                    newobs[i + 1] = obs[i];
                obs = newobs;
            }
            ThreadCheck();
            if (IsSubsetSend(obs[0]))
            {
                usesSubsetSend = true;
                doOrderedSubsetSend(obs);
            }
            else
            {
                if (usesSubsetSend && (IsisSystem.Debug & IsisSystem.WARNABOUTSUBSETS) != 0)
                    Isis.WriteLine("WARNING: In group <" + gname + "> application is mixing OrderedSubsetSends with OrderedSend.  The relative order is not guaranteed");
                OrderedSendWith((Action<int, int, bool, Msg>)delegate(int vid, int mid, bool flag, Msg m) { doSend(vid, mid, m.nRaw, flag, Isis.ORDEREDSEND, m); }, obs);
            }
            IsisCallDone();
        }
        private static bool IsSubsetSend(object obj)
        {
            Type t = obj.GetType();
            return t.Equals(typeof(List<Address>)) || (t.IsGenericType && t.GetGenericTypeDefinition() == typeof(QueryKey<>)) || (t.Equals(typeof(Msg)) && ((Msg)obj).destList != null);
        }

        private void doOrderedSubsetSend(object[] obs)
        {
            bool copyDown = false;
            List<Address> dests;
            Type t = obs[0].GetType();
            if (t.Equals(typeof(List<Address>)))
            {
                dests = (List<Address>)obs[0];
                copyDown = true;
            }
            else if (t.Equals(typeof(Msg)) && ((Msg)obs[0]).destList != null)
                dests = ((Msg)obs[0]).destList;
            else if (t.IsGenericType && t.GetGenericTypeDefinition() == typeof(QueryKey<>))
            {
                dests = ((QKD)obs[0]).GetDests(this);
                copyDown = true;
            }
            else
                throw new IsisException("doOrderedSubsetSend: can't identify the subset destination list");
            if (copyDown)
            {
                object[] newObs = new object[obs.Length - 1];
                for (int n = 0; n < obs.Length - 1; n++)
                    newObs[n] = obs[n + 1];
                obs = newObs;
            }
            Msg m = new Msg(obs);
            SetMsgIds(m, false, true);
            List<long> ts = new List<long>();
            List<Address> who = new List<Address>();
            if ((IsisSystem.Debug & IsisSystem.ORDEREDSEND) != 0)
                Isis.WriteLine("doOrderedSubsetSend[" + Address.VectorToString(dests.ToArray()) + "]... sending " + m.sender + "::" + m.vid + ":" + m.msgid + "... will wait for " + dests.Count() + " responses");
            FlowControl.FCBarrierCheck(dests);
            int nr = doQuery(dests.Count(), new Timeout(Isis.ISIS_DEFAULTTIMEOUT * 3 / 2, Timeout.TO_FAILURE), dests, Isis.ORDEREDSEND, dests, myTS, m, EOL, ts, who);
            if ((IsisSystem.Debug & IsisSystem.ORDEREDSEND) != 0)
                Isis.WriteLine("doOrderedSubsetSend... " + m.sender + "::" + m.vid + ":" + m.msgid + "... got " + nr + " responses");
            if (nr > 0)
            {
                long cts = ts[0];
                Address cwho = who[0];
                for (int n = 1; n < nr; n++)
                    if (ts[n] == cts)
                    {
                        cts = ts[n];
                        if (who[n].CompareTo(cwho) > 0)
                            cwho = who[n];
                    }
                    else if (ts[n] > cts)
                    {
                        cts = ts[n];
                        cwho = who[n];
                    }
                if ((IsisSystem.Debug & IsisSystem.ORDEREDSEND) != 0)
                {
                    Isis.WriteLine("doOrderedSubsetSend... got " + nr + " responses for " + m.sender + "::" + m.vid + ":" + m.msgid);
                    Isis.WriteLine("... send [" + Address.VectorToString(dests.ToArray()) + "] commit time @ " + cts + "::" + cwho);
                }
                if (cts > myTS)
                    myTS = cts;
                doSend(false, false, dests, Isis.ORDEREDSEND, m.sender, m.vid, m.msgid, cts, cwho);
            }
        }

        private void OrderedSendWith(Action<int, int, bool, Msg> theSend, object[] obs)
        {
            if ((flags & G_ISLARGE) != 0)
            {
                Send(obs);
                return;
            }
            if (myLoggingFcn != null)
                myLoggingFcn(IL_ORDEREDSEND, IL_START, Isis.my_address, newLoggingId(), obs);
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            if (obs.Length != 1 || !obs[0].GetType().Equals(typeof(Msg)))
                cbCheck(obs);
            RateLimit(obs.Length);
            FlowControl.FCBarrierCheck();
            Msg m;
            if (obs.Length == 1 && obs[0].GetType().Equals(typeof(Msg)))
                m = (Msg)obs[0];
            else
            {
                m = new Msg(obs);
                try
                {
                    Wedged.WaitOne();
                    using (new LockAndElevate(CommitLock))
                    using (new LockAndElevate(ViewLock))
                    {
                        m.vid = theView == null ? -1 : theView.viewid;
                        m.msgid = nextMsgid++;
                        m.nRaw = nRaw;
                        nRaw = 0;
                    }
                }
                finally
                {
                    Wedged.Release();
                }
            }
            m.dest = gaddr;
            theSend(m.vid, m.msgid, false, m);
        }

        delegate void MergeSafeSendReplies(int[] timestamp, Address[] who);

        /// <summary>
        /// The slowest but strongest of the Isis multicast primitives, offering a totally ordered, durable multicast matching the Paxos semantics.
        /// </summary>
        /// <param name="obs">A variable-length argument list specifying a request ID and a set of arguments that should match some handler.</param>
        /// <throws>IsisSafeSendException if the group size drops below the SafeSendThreshold</throws>
        /// <remarks>
        /// The slowest but strongest of the Isis multicast primitives, offering a totally ordered, durable multicast matching the Paxos semantics
        /// and suitable for use in State Machine Replication or Transactional Database settings where the very strongest possible properties
        /// are needed, even at substantial performance cost.
        /// 
        /// This form of Send is costly, and might not be the right choice: the properties are very powerful, but they aren't cheap. 
        /// The cost is even higher if SafeSend is used in a large group from any member except the rank-0 process (this cost could
        /// be reduced but the needed code isn't trivial and this version of Isis is just not optimized for SafeSend by non-rank0 members of large groups)
        /// 
        /// Before using SafeSend we recommend that you consider using the mostly-safe Send primitives, but then call Flush before replying
        /// to the end-user.  This will usually be just as safe and yet much faster for many purposes.  We've written a paper on this topic.
        /// 
        /// Basically, maps to a two-phase commit: the first phase is via IP multicast, but safesendThreshold group members must ack before delivery occurs.
        /// The default for safesendThreshold is ALL, but you can set a lower value using SetSafeSendThreshold to some other value, like 2 or 3.
        /// The sender will count as one of the the acking members, so in fact Isis actually waits for safesendThreshold-1 replies.
        /// If a view change occurs, any pending SafeSend messages that have not yet been delivered are flushed (hence reach
        /// all members) and then delivered as part of the new view COMMIT protocol.
        /// 
        /// The SafeSend primitive is synchronized with respect to membership changes in accordance with the virtual synchrony model, which is a strong and
        /// useful property.  The first parameter is an identifier for the message handler to invoke (a small integer) and the remai ning
        /// are the arguments, which are objects of types known to Isis.  An exception will be thrown if the group doesn't have a message handler for
        /// the specified request id, and matching the number and types of objects you specified.
        /// 
        /// Read more about the <it>virtual synchrony model</it> to learn about the options
        /// and how to pick the fastest mechanism for the setting in which your application will run.
        /// </remarks>
        public void SafeSend(params object[] obs)
        {
            if (!IsisCallStart())
                return;
            if ((flags & G_ISLARGE) != 0)
                throw new IsisException("Safesend: Not supported in large groups");
            ThreadCheck();
            if (myLoggingFcn != null)
                myLoggingFcn(IL_SAFESEND, IL_START, Isis.my_address, newLoggingId(), obs);
            TypeCheck(obs);
            RateLimit(obs.Length);
            FlowControl.FCBarrierCheck();
            doSafeSend(obs);
            IsisCallDone();
        }

        /// <summary>
        /// This method is used to specify how many members of a group need to acknowledge a SafeSend multicast before it can be delivered.
        /// For example, if the value specified is 3, then once a copy of the message is at 3 members, delivery occurs (even if the group has 1000 members).
        /// </summary>
        /// <param name="value">The number of members SafeSend will wait for</param>
        /// <remarks>SafeSend waits for replies from the members ranked 0..SafeSendThreshold-1.  This parameter functions exactly as does the Paxos
        /// "number of acceptors" parameter (continuing the analogy, the full set of members are "listeners")</remarks>
        public void SetSafeSendThreshold(int value)
        {
            safeSendThreshold = value;
        }

        internal int GetSafeSendThreshold()
        {
            View theView;
            using (new LockAndElevate(ViewLock))
                theView = this.theView;
            if (HasFirstView)
                if (safeSendThreshold == ALL)
                    return theView.members.Length;
                else if (theView.members.Length < safeSendThreshold)
                    return -1;
                else
                    return safeSendThreshold;
            return -1;
        }

        /// <summary>
        /// Specifies a method that will be used to ensure the durability of a SafeSend.  If not specified, in-memory caching at the acceptors is used
        /// </summary>
        /// <param name="theMethod">A method implementing the durabilityMethod API</param>
        public void SetDurabilityMethod(durabilityMethod theMethod)
        {
            safeSendDurabilityMethod = theMethod;
        }

        private void doSafeSend(object[] obs)
        {
            if (theView.nLive() < safeSendThreshold)
                throw new IsisSafeSendException("Group<" + gname + ">: size dropped below SafeSend threshold=" + safeSendThreshold);
            int maxTime = Int32.MinValue;
            Address maxAddr = Isis.my_address;
            cbCheck(obs);
            Msg m;
            if (obs.Length == 1 && obs[0].GetType().Equals(typeof(Msg)))
                m = (Msg)obs[0];
            else
            {
                m = new Msg(obs);
                try
                {
                    Wedged.WaitOne();
                    using (new LockAndElevate(groupLock))
                    using (new LockAndElevate(CommitLock))
                    {
                        m.vid = theView.viewid;
                        m.msgid = nextMsgid++;
                        m.nRaw = nRaw;
                        nRaw = 0;
                    }
                }
                finally
                {
                    Wedged.Release();
                }
            }
            // Timeout needs to be long because we might encounter the disklogger in the middle of garbage collection, which involves copying files and can take a while
            Msg.InvokeFromBArrays(QueryToBAFromSystem(m.vid, m.msgid, m.nRaw, safeSendThreshold, new Timeout(Isis.ISIS_DEFAULTTIMEOUT * 4, Timeout.TO_FAILURE, "SAFESEND"), Isis.SAFESEND, Isis.my_address, m.msgid, m), (MergeSafeSendReplies)delegate(int[] timestamp, Address[] who)
            {
                if (timestamp.Length < safeSendThreshold)
                    throw new IsisSafeSendException("Group<" + gname + ">: size dropped below SafeSend threshold=" + safeSendThreshold);
                for (int n = 0; n < timestamp.Length; n++)
                    if (timestamp[n] > maxTime || (timestamp[n] == maxTime && who[n].GetHashCode() > maxAddr.GetHashCode()))
                    {
                        maxTime = timestamp[n];
                        maxAddr = who[n];
                    }
            });
            // By using Send instead of doSend, this code deliberately allows the Send to be blocked if the group
            // is wedged for a membership change (doSend ignores that kind of wedging).  In such cases upon receipt,
            // the corresponding message will already have been delivered. But we don't want flow control to kick in.
            NonFlowControlledSend(Isis.SAFEDELIVER, Isis.my_address, m.msgid, maxTime, maxAddr);
        }

        /// <summary>
        /// The cheapest and fastest of the Isis multicast primitives, offering a FIFO (by sender) but otherwise unordered, non-durable multicast.
        /// </summary>
        /// <param name="obs">A variable-length argument list specifying a request ID and a set of arguments that should match some handler.</param>
        /// <remarks> 
        /// The cheapest and fastest of the Isis multicast primitives, offering a FIFO (by sender) but otherwise unordered, non-durable multicast.
        /// 
        /// This cheap form of Send is usually, but not always, the right choice.  Read more about the <it>virtual synchrony model</it> to learn about the options
        /// and how to pick the fastest mechanism for the setting in which your application will run.
        /// 
        /// Basically, maps to an IP multicast, but with reliability and ordering on a per-sender basis, and delivered as soon as it arrives.
        /// Nonetheless, synchronized with respect to membership changes in accordance with the virtual synchrony model, which is a strong and
        /// useful property.  The first parameter is an identifier for the message handler to invoke (a small integer) and the remaining
        /// are the arguments, which are objects of types known to Isis.  An exception will be thrown if the group doesn't have a message handler for
        /// the specified request id, and matching the number and types of objects you specified.
        /// </remarks>
        public void Send(params object[] obs)
        {
            _Send(true, obs);
        }

        private void NonFlowControlledSend(params object[] obs)
        {
            _Send(false, obs);
        }

        private void _Send(bool okToBlock, object[] obs)
        {
            if (!IsisCallStart())
                return;
            obs = subsetShortcut(obs);
            if (okToBlock)
                ThreadCheck();
            if (myLoggingFcn != null)
                myLoggingFcn(IL_SEND, IL_START, Isis.my_address, newLoggingId(), obs);
            if (okToBlock)
                RateLimit(obs.Length);
            List<Address> dests = null;
            if (IsSubsetSend(obs[0]))
                dests = obs[0].GetType().Equals(typeof(List<Address>)) ? ((List<Address>)obs[0]) : ((QKD)obs[0]).GetDests(this);
            if (okToBlock)
                FlowControl.FCBarrierCheck(dests);
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            doSend(false, false, obs);
            IsisCallDone();
        }

        private static object[] subsetShortcut(object[] obs)
        {
            if (obs.Length > 1 && obs[0].GetType().Equals(typeof(int)) && IsSubsetSend(obs[1]))
            {
                object[] newobs = new object[obs.Length + 1];
                newobs[0] = obs[1];
                for (int i = 0; i < obs.Length; i++)
                    newobs[i + 1] = obs[i];
                obs = newobs;
            }
            return obs;
        }

        internal void ThreadCheck()
        {
            if (this != Isis.ORACLE && this != Isis.ISISMEMBERS && (!IsisSystem.IsisActive || IsisSystem.IsisRestarting || !GroupOpen))
                throw new IsisException("Group multicast operation was initiated before the system was fully operational or before the group Join completed");
            if (Thread.CurrentThread == groupIPMCReaderThread)
                Isis.WriteLine("WARNING: <" + gname + ">.Send()/g.Query() or some multicast primitive was called from the incoming multicasts delivery thread (high risk of deadlocks)");
        }

        /// <summary>
        /// An unreliable version of Send.
        /// </summary>
        /// <param name="obs">A variable-length argument list specifying a request ID and a set of arguments that should match some handler.</param>
        /// <remarks> 
        /// The RawSend API is provided for advanced users only.  The multicast is sent without retransmission in the event of failure and
        /// is not acknowledged by the receiver.  RawSend is currently disabled for Large Group cases (it maps to reliable Send)
        /// </remarks>
        public void RawSend(params object[] obs)
        {
            if (!IsisCallStart())
                return;
            ThreadCheck();
            if (myLoggingFcn != null)
                myLoggingFcn(IL_SEND, IL_START, Isis.my_address, newLoggingId(), obs);
            RateLimit(obs.Length);
            FlowControl.FCBarrierCheck();
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            doSend(false, (flags & G_ISLARGE) == 0, obs);
            IsisCallDone();
        }

        /// <summary>
        /// An Isis multicast that runs nearly as fast as Send, but with a stronger "casuality-preserving" ordering property.
        /// </summary>
        /// <param name="obs">A variable-length argument list specifying a request ID and a set of arguments that should match some handler.</param>
        /// <remarks> 
        /// Imagine that Send(X) is performed by member A, and then member B, receiving X, issues Send(Y).  With the FIFO property of Send, nothing prevents
        /// member C from receiving Y before X!  CausalSend guarantees that if X "happens before" Y, then CausalSend(X) will be delivered before CausalSend(Y)
        /// 
        /// For most purposes, Send or Causal send is the best choice, followed by a call to Flush() prior to responding to external clients.  
        /// Read more about the <it>virtual synchrony model</it> to learn about the options
        /// and how to pick the fastest mechanism for the setting in which your application will run.
        /// 
        /// Send and CausalSend both map to an IP multicast or a tunneled application-implemented multicast, and CausalSend will usually be delivered as soon as it arrives.
        /// CausalSend is synchronized with respect to membership changes in accordance with the virtual synchrony model, which is a strong and
        /// useful property.  The first parameter is an identifier for the message handler to invoke (a small integer) and the remaining
        /// are the arguments, which are objects of types known to Isis.  An exception will be thrown if the group doesn't have a message handler for
        /// the specified request id, and matching the number and types of objects you specified.
        /// </remarks>
        public void CausalSend(params object[] obs)
        {
            if (!IsisCallStart())
                return;
            ThreadCheck();
            if ((flags & G_ISLARGE) != 0)
                Send(obs);
            else if (theView.members.Length > 5)
                // In larger groups the VT scheme doesn't work well; use OrderedSend instead
                OrderedSend(obs);
            else
            {
                int[] myVT;
                int theVid;
                using (new LockAndElevate(ViewLock))
                {
                    theVid = theView.viewid;
                    myVT = new int[theView.myVT.Length];
                    Array.Copy(theView.myVT, myVT, theView.myVT.Length);
                    myVT[theView.GetMyRank()] = theView.myClock++;
                }
                if ((IsisSystem.Debug & IsisSystem.CAUSALDELIVERY) != 0)
                    Isis.WriteLine("Sending a causal multicast in vid=" + theVid + ", with VT=" + VTtoString(myVT));
                Send(Isis.CAUSALSEND, theVid, myVT, new Msg(obs));
            }
            IsisCallDone();
        }

        internal static void doMultiSend(List<Group> glist, bool fromOracle, params object[] obs)
        {
            foreach (Group g in glist)
            {
                g.RateLimit(obs.Length);
                g.TypeCheck(obs);
                g.doSend(fromOracle, false, obs);
            }
        }

        private LockObject RelayedLGSendsLock = new LockObject("RelayedLGSendsLock");
        private List<Msg> RelayedLGSends = new List<Msg>();
        private Address prevLGOwner;
        private bool LGSetupDone = false;

        internal string LGRelayGetState()
        {
            string s = "";
            using (new LockAndElevate(RelayedLGSendsLock))
            {
                if (RelayedLGSends.Count() == 0)
                    return s;
                s = "LG Relayed Sends queue:\r\n";
                foreach (Msg m in RelayedLGSends)
                    s += "  --  " + m + "\r\n";
            }
            return s;
        }

        internal void LGSetup()
        {
            if (LGSetupDone)
                return;
            LGSetupDone = true;
            tokenInfo outerToken;
            using (new LockAndElevate(TokenLock))
                outerToken = this.theToken;
            if (outerToken != null)
                prevLGOwner = outerToken.groupOwner;
            // Note that the peek-ahead logic for messages watches for RELAYSEND and expects the ViewDelta vector to be the first (and only) argument... If this is changed, that also needs to change
            doRegister(Isis.RELAYSEND, (Action<Isis.ViewDelta[]>)delegate(Isis.ViewDelta[] vds)
            {
                tokenInfo theToken;
                View theView = null;
                if (!HasFirstView)
                    using (new LockAndElevate(CommitLock))
                        Isis.CommitGVUpdates(this, vds, ref theView); ;
                using (new LockAndElevate(TokenLock))
                using (new LockAndElevate(ViewLock))
                {
                    theToken = this.theToken;
                    theView = this.theView;
                }
                vds = vds.Select(vd => vd).Where(vd => vd.gaddr.Equals(this.gaddr) && vd.prevVid >= (theView == null ? -1 : theView.viewid)).ToArray();
                if (vds.Length == 0)
                    return;
                if ((IsisSystem.Debug & (IsisSystem.RELAYLOGIC | IsisSystem.TOKENLOGIC)) != 0)
                {
                    Isis.WriteLine("Large Group owner relaying COMMIT for a vector of view deltas in <" + gname + ">");
                    foreach (Isis.ViewDelta vd in vds)
                        Isis.WriteLine("   " + vd);
                }
                if (theView == null || theToken == null)
                    throw new IsisException("Large group had a total failure just as I was joining.");

                theToken.applyViewDeltas(this, vds);
                if (myFirstLeadershipView == 0)
                    return;
                becomeGroupOwner();
                foreach (Isis.ViewDelta vd in vds)
                    if (vd.prevVid >= theView.viewid)
                    {
                        if ((IsisSystem.Debug & (IsisSystem.GROUPEVENTS | IsisSystem.RELAYLOGIC | IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                            Isis.WriteLine("In RELAYSEND relaying COMMIT in <" + gname + "> for ViewDelta " + vd);
                        using (new LockAndElevate(theToken.slock))
                        {
                            if (theToken.unstableVIDMID == 0)
                            {
                                theToken.unstableVIDMID = nextMsgid;
                                theToken.unstableVID = theToken.viewid;
                            }
                        }
                        doSend(false, false, Isis.COMMIT, vds);
                        break;
                    }

                List<Msg> toRelay;
                using (new LockAndElevate(RelayedLGSendsLock))
                {
                    toRelay = RelayedLGSends;
                    RelayedLGSends = new List<Msg>();
                }
                foreach (Msg m in toRelay)
                    doSend(false, false, m);

                // Asynchronously send off a message to the ORACLE: I'm stable up to whatever the current viewid shows
                int stableVID;
                using (new LockAndElevate(theToken.slock))
                    stableVID = theToken.stableVID;
                Isis.ORACLE.doSend(false, false, Isis.ISSTABLE, gaddr, stableVID);
            });

            // Register the flush aggregator for this group
            doRegisterAggregator<FlushAggKey, bool>((Aggregator<FlushAggKey, bool>)delegate(FlushAggKey key, bool lV, bool dV)
            {
                return lV & dV;
            });
        }

        internal void doUnorderedSend(params object[] obs)
        {
            if (obs == null || obs.Length < 1)
                throw new ArgumentNullException("Isis.Group.Cast");
            if (obs.Length == 1 && obs[0].GetType().Equals(typeof(object[])))
                obs = ((object[])obs[0]);
            doTheSend(false, false, false, Msg.UNORDERED, obs);
        }

        internal void doSendNotFromOracle(params object[] obs)
        {
            doSend(false, false, obs);
        }

        internal void doRawSendNotFromOracle(params object[] obs)
        {
            doSend(false, true, obs);
        }

        internal void doSendFromOracle(params object[] obs)
        {
            doSend(true, false, obs);
        }

        internal void doSend(bool isFromOracle, bool isRaw, params object[] obs)
        {
            if (obs == null || obs.Length < 1)
                throw new ArgumentNullException("Isis.Group.Cast");
            if (obs.Length == 1 && obs[0].GetType().Equals(typeof(object[])))
                obs = ((object[])obs[0]);
            doTheSend(isFromOracle, isRaw, false, Msg.FIFOCAST, obs);
        }

        internal void doSend(int vid, int msgid, int nRaw, bool isFromOracle, params object[] obs)
        {
            if (obs == null || obs.Length < 1)
                throw new ArgumentNullException("Isis.Group.Cast");
            if (obs.Length == 1 && obs[0].GetType().Equals(typeof(object[])))
                obs = ((object[])obs[0]);
            doTheSend(vid, msgid, nRaw, isFromOracle, true, false, Msg.FIFOCAST, obs);
        }

        internal void doSendRaw(params object[] obs)
        {
            if (obs == null || obs.Length < 1)
                throw new ArgumentNullException("Isis.Group.Cast");
            if (obs.Length == 1 && obs[0].GetType().Equals(typeof(object[])))
                obs = ((object[])obs[0]);
            doTheSend(false, true, true, Msg.FIFOCAST, obs);
        }

        private void doTheSend(bool sentByOracle, bool isRaw, bool isBeacon, byte type, object[] obs)
        {
            doTheSend(Msg.UNINITIALIZED, Msg.UNINITIALIZED, 0, sentByOracle, isRaw, isBeacon, type, obs);
        }

        private void doTheSend(int vid, int msgid, int nRaw, bool sentByOracle, bool isRaw, bool isBeacon, byte type, object[] obs)
        {
            Msg m;
            // Loops while it finds the group wedged (e.g. membership is changing), then does the requested send
            int waitingTime = 0;
            for (int retry = 0; waitingTime < Isis.ISIS_DEFAULTTIMEOUT * 4 && (IsisSystem.IsisActive || !IsisSystem.IsisWasActive); retry++)
            {
                if (retry > 0)
                {
                    int howLong = retry < 10 ? 25 : 250;
                    Thread.Sleep(howLong);
                    waitingTime += howLong;
                }
                // Perhaps non-ideal but the idea here is that from when we first assign message id's to a multicast until we send the
                // last fragment, we hold the SIFLock, preventing anyone else from touching the multicast send id# counter
                // The down side is that for Isis2, this is a fairly long-held lock and may be implicated in a multi-thread deadlock
                // involving the CommitLock
                using (new LockAndElevate(SIFLock))
                {
                    int xvid = -1;
                    using (new LockAndElevate(ViewLock))
                        if (theView != null)
                            xvid = theView.viewid;
                    if (!isRaw && (flags & G_WEDGED) != 0)
                    {
                        if ((IsisSystem.Debug & IsisSystem.GROUPEVENTS) != 0)
                            Isis.WriteLine("doTheSend: forced to loop because <" + gname + "> vid " + xvid + " is wedged");
                        continue;
                    }
                    if (obs[0].GetType().Equals(typeof(Msg)) && ((Msg)obs[0]).destList != null)
                    {
                        m = (Msg)obs[0];
                        foreach (Address dest in m.destList)
                            doP2PSend(dest, true, m);
                        return;
                    }
                    if (IsSubsetSend(obs[0]))
                    {
                        List<Address> dests;
                        if (obs[0].GetType().Equals(typeof(List<Address>)))
                            dests = (List<Address>)obs[0];
                        else
                            dests = ((QKD)obs[0]).GetDests(this);
                        obs = fixObs(obs);
                        cbCheck(obs);
                        m = new Msg(obs);
                        SetMsgIds(m, false, true);
                        foreach (Address dest in dests)
                            doP2PSend(dest, true, m);
                        return;
                    }
                    if ((IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0)
                        Isis.WriteLine("doSend: GroupOpen " + GroupOpen + ", sendByOracle=" + sentByOracle + ", isRaw=" + isBeacon + ", type=" + type);
                    if (!GroupOpen && !isBeacon)
                        return;
                    if (obs.Length == 1 && obs[0].GetType().Equals(typeof(Msg)))
                        m = (Msg)obs[0];
                    else
                    {
                        cbCheck(obs);
                        m = new Msg(obs);
                    }
                    if (sentByOracle)
                        m.flags |= Msg.SENTBYORACLE;
                    if ((flags & G_ISLARGE) != 0)
                    {
                        tokenInfo theToken;
                        using (new LockAndElevate(TokenLock))
                            theToken = this.theToken;
                        if (theToken != null && theToken.IAmLgOwner == false)
                        {
                            if (m.vid != -1 && m.msgid != -1)
                                using (new LockAndElevate(RelayedLGSendsLock))
                                    RelayedLGSends.Add(m);
                            doP2PSend(theToken.groupOwner, true, Isis.RELAYSEND, m);
                            return;
                        }
                    }
                    if (isBeacon)
                    {
                        // Special, used currently only by ORACLE BeaconTask
                        m.vid = 0;
                        m.msgid = -1;
                    }
                    else if (vid != Msg.UNINITIALIZED)
                    {
                        m.vid = vid;
                        m.msgid = msgid;
                        m.nRaw = nRaw;
                    }
                    else if (m.vid == Msg.UNINITIALIZED)
                        SetMsgIds(m, sentByOracle, isRaw);
                    if ((flags & G_SECURE) != 0 && (type == Msg.ISGRPP2P || type == Msg.ISRAWGRPP2P || type == Msg.FIFOCAST || type == Msg.RAWFIFOCAST || type == Msg.UNORDERED || type == Msg.ISREPLY))
                        cipherMsg(m);
                    if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.VIEWWAIT)) != 0)
                        Isis.WriteLine("ReliableSender.SendGroup to <" + gname + ">... type=" + Msg.mtypes[type] + ", Msg=" + m);
                    ReliableSender.SendGroup(type, this, m.vid, m.msgid, (byte)(m.flags & ~Msg.CIPHER), Msg.toBArray(m), m.nRaw);
                }
                return;
            }
            if (!IsisSystem.IsisActive)
                throw new IsisShutdown("doTheSend");
            // NOTE: The timeout has to be long because (1) flush can be slow, and worse than that, someone may have just
            // crashed without a graceful shutdown; if so, detecting the failure will take something like 2*ISIS_DEFAULTTIMEOUT time
            throw new IsisException("doTheSend: Group stuck in a wedged state for " + waitingTime / 1000 + " seconds");
        }

        internal void SetMsgIds(Msg m, bool sentByOracle, bool isRaw)
        {
            ILock lgb;
            tokenInfo theToken;
            using (new LockAndElevate(TokenLock))
                theToken = this.theToken;
            if (myLoggingFcn != null)
                m.Lid = lookupLoggingId();
            if ((flags & G_ISLARGE) != 0)
            {
                if (theToken != null)
                {
                    using (new LockAndElevate(theToken.FlushingBarrierLock))
                        lgb = theToken.FlushingBarrier;
                    if (lgb != null)
                    {
                        if ((IsisSystem.Debug & IsisSystem.FLUSHING) != 0)
                            Isis.WriteLine("Before BarrierWait in SetMsgIds");
                        lgb.BarrierWait();
                        if ((IsisSystem.Debug & IsisSystem.FLUSHING) != 0)
                            Isis.WriteLine("After BarrierWait in SetMsgIds");
                    }
                }
            }
            string which = "?";
            if (!sentByOracle || this == Isis.ORACLE)
            {
                if (HasFirstView)
                {
                    // Blocks if group view is currently changing
                    try
                    {
                        // Once we've assigned a message id, this member is committed to delivering this message in the current view
                        Wedged.WaitOne();
                        using (new LockAndElevate(CommitLock))
                        using (new LockAndElevate(ViewLock))
                        {
                            m.vid = theView.viewid;
                            m.msgid = nextMsgid++;
                            // Each message tells how many raw messages, sent in a row, it immediately follows
                            if (isRaw)
                                m.nRaw = nRaw++;
                            else
                            {
                                m.nRaw = nRaw;
                                nRaw = 0;
                            }
                        }
                        which = "HasFirstView";
                    }
                    finally
                    {
                        Wedged.Release();
                    }
                }
                else
                {
                    m.vid = 0;
                    m.msgid = -1;
                    which = "NoFirstView";
                }
            }
            else if ((flags & G_ISLARGE) == 0)
            {
                Group tp = TrackingProxyLookup(gaddr);
                if (tp != null && tp.HasFirstView)
                {
                    using (new LockAndElevate(CommitLock))
                    using (new LockAndElevate(tp.ViewLock))
                    {
                        m.vid = tp.theView.viewid;
                        m.msgid = tp.nextMsgid++;
                        m.nRaw = nRaw;
                        nRaw = 0;
                    }
                    which = "TP/HasFirstView";
                }
                else
                {
                    m.vid = 0;
                    m.msgid = -1;
                    which = "TP/NoFirstView";
                }
            }
            else
                throw new IsisException("Oracle is trying to send in a large group: <" + gname + ">");
            m.dest = gaddr;
            if ((IsisSystem.Debug & IsisSystem.MSGIDS) != 0)
                Isis.WriteLine("Set msgids(SentByOracle=" + sentByOracle + ") " + which + ": " + m);
        }

        /// <summary>
        /// Flushes any unstable messages, pauses until the operation completes, then returns.
        /// </summary>
        /// <remarks>
        /// When using the non-Paxos multicast and query options, conditions can arise in which the user needs to know that past operations
        /// on the group have stabilized before some next action (such as checkpointing the group) can occur.  For this, call Flush.
        /// To learn more, read about the <it>virtual synchrony model</it> as implemented by Isis.
        /// </remarks>
        public void Flush()
        {
            Flush(int.MaxValue);
        }

        /// <summary>
        /// Flushes unstable messages until k copies have definitely reached their destinations
        /// </summary>
        /// <param name="k">A "stability threshold" similar to the Paxos stability threshold parameter.</param>
        public void Flush(int k)
        {
            if (!IsisSystem.IsisActive)
                return;
            checkGroupIsOpen(this);
            if ((flags & G_ISLARGE) == 0 && k > 0)
                ReliableSender.waitForStability(k);
            else
                doFlush(k, new Isis.UnstableList[0], new Address[0]);
        }

        private static void checkGroupIsOpen(object g)
        {
            if (g == null)
                docheckGroup(null);
            else if (g.GetType().Equals(typeof(Group)))
                docheckGroup((Group)g);
            else if (g.GetType().Equals(typeof(List<Group>)))
                foreach (Group grp in (List<Group>)g)
                    docheckGroup((Group)grp);
        }

        private static void docheckGroup(Group g)
        {
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            if (g != null && !g.GroupOpen && g.WasOpen)
                throw new IsisException("An operation was attempted on group <" + g.gname + ">, but this group is closed");
        }

        internal void doFlush(int k, Isis.UnstableList[] usl, Address[] leaving)
        {
            List<Msg> mustSend;
            using (Semaphore CPSSema = new Semaphore(0, int.MaxValue))
            {
                int cpscnt;
                startFlush(k, usl, out mustSend, out cpscnt, CPSSema);
                endFlush(leaving, mustSend, cpscnt, CPSSema);
            }
        }

        internal void startFlush(int k, Isis.UnstableList[] usl, out List<Msg> mustSend, out int cpscnt, Semaphore CPSSema)
        {
            mustSend = new List<Msg>();
            using (new LockAndElevate(UnstableLock))
            {
                foreach (Isis.UnstableList us in usl)
                {
                    if (us.flusher.Equals(Isis.my_address) == false)
                        continue;
                    else
                    {
                        foreach (Msg m in Unstable)
                            if (m.gaddr.Equals(us.gaddr) && m.sender.Equals(us.sender) && m.vid == us.vid && m.msgid >= us.mid_low && m.msgid <= us.mid_hi)
                            {
                                Unstable.Remove(m);
                                mustSend.Add(m);
                                break;
                            }
                    }
                }
            }
            cpscnt = 0;
            using (new LockAndElevate(ReliableSender.PendingSendBufferLock))
            {
                foreach (ReliableSender.MsgDesc md in ReliableSender.PendingSendBuffer)
                    if (md.group == this)
                        lock (md)
                        {
                            if (md.CPSList != null)
                            {
                                md.CPSList.Add(CPSSema);
                                ++cpscnt;
                            }
                        }
                foreach (ReliableSender.MsgDesc lgmd in ReliableSender.LgPendingSendBuffer)
                    if (lgmd.group == this)
                        lock (lgmd)
                        {
                            if (lgmd.CPSList != null)
                            {
                                lgmd.CPSList.Add(CPSSema);
                                ++cpscnt;
                            }
                        }
            }
        }

        internal void endFlush(Address[] leaving, List<Msg> mustSend, int cpscnt, Semaphore CPSSema)
        {
            foreach (Msg m in mustSend)
            {
                if ((flags & G_SECURE) != 0)
                    cipherMsg(m);
                ReliableSender.SendGroup(m.type, this, m, m.sender.isMyAddress());
            }
            ReliableSender.PendingSendCleanup(leaving);
            ReliableSender.CompletePendingSends(this, cpscnt, CPSSema);
        }

        internal class FragInfo
        {
            internal Address sender;
            internal int fragId;
            internal bool[] gotFrag;
            internal bool iscomp;
            internal int nFragsRemaining;
            internal byte[] body;
            internal Msg TrueMsg;
            internal LockObject Lock = new LockObject("FragInfo.Lock");

            internal FragInfo(Address s, int fid, long tl, int nf, bool ic)
            {
                sender = s;
                fragId = fid;
                body = new byte[tl];
                nFragsRemaining = nf;
                gotFrag = new bool[nf];
                iscomp = ic;
                if (!Isis.BigTimeouts && tl > 500000)
                {
                    // Switch to big-timeout mode if incoming object is larger than 500KB
                    Isis.BigTimeouts = true;
                    Isis.ISIS_DEFAULTTIMEOUT = Isis.ISIS_DEFAULTTIMEOUT * 2;
                }
            }
        }

        internal static string deFragState()
        {
            string s = "Defragmentation in progress:\r\n";
            using (new LockAndElevate(dfLock))
            {
                if (dfList.Count() == 0)
                    return "";
                foreach (FragInfo fi in dfList)
                {
                    using (new LockAndElevate(fi.Lock))
                    {
                        string got = " ";
                        foreach (bool b in fi.gotFrag)
                            got += b ? "+ " : "- ";
                        s += "  [sender=" + fi.sender + ", fragId=" + fi.fragId + ",, nFragsRemaining=" + fi.nFragsRemaining + ", got={" + got + "}, final length will be " + fi.body.Length + "]\r\n";
                    }
                }
                return s;
            }
        }

        internal static LockObject dfLock = new LockObject("dfLock");
        internal static LockObject sendInFragsLock = new LockObject("sendInFragsLock");
        internal static List<FragInfo> dfList = new List<FragInfo>();
        internal static int nextFid;

        internal static FragInfo deFragLookup(Group g, Address sender, int fid, long tl, int nf, bool iscomp)
        {
            using (new LockAndElevate(dfLock))
                foreach (FragInfo dfi in dfList)
                    if (dfi.sender.Equals(sender) && dfi.fragId == fid)
                        return dfi;
            using (new LockAndElevate(Isis.RIPLock))
                if (Isis.RIPList.Contains(sender))
                    return null;
            FragInfo fi = new FragInfo(sender, fid, tl, nf, iscomp);
            using (new LockAndElevate(dfLock))
                dfList.Add(fi);
            return fi;
        }

        internal static void deFragGotFrag(Group g, Address sender, int fragId, long trueLen, int nFrags, int fragN, bool ic, byte[] frag)
        {
            FragInfo fi = deFragLookup(g, sender, fragId, trueLen, nFrags, ic);
            if ((IsisSystem.Debug & IsisSystem.FRAGER) != 0)
                Isis.WriteLine("deFragmenter got a fragment for a " + trueLen + " byte object; senderId=" + Isis.my_address + ", fid=" + fragId + ", this was fragment " + fragN + (fi == null ? " (** SENDER ON RIP LIST: IGNORING **)" : ""));
            if (fi == null)
                return;
            bool doDelivery = false;
            List<FragInfo> toRemove = new List<FragInfo>();
            using (new LockAndElevate(fi.Lock))
            {
                if (fi.gotFrag[fragN])
                    return;
                fi.gotFrag[fragN] = true;
                Array.Copy(frag, 0, fi.body, fragN * Isis.ISIS_FRAGLEN, frag.Length);
                if (--fi.nFragsRemaining == 0)
                    if (fi.TrueMsg != null)
                        doDelivery = true;
                    else
                        toRemove.Add(fi);
            }
            using (new LockAndElevate(dfLock))
                foreach (FragInfo rfi in toRemove)
                    dfList.Remove(rfi);
            if (doDelivery)
                fragDoDelivery(g, fi);
        }

        internal static void deFragRdv(Group g, Msg m)
        {
            Msg outer = (Msg)Msg.BArrayToObjects(m.payload, typeof(Msg))[0];
            object[] objs = Msg.BArrayToObjects(outer.payload, typeof(Address), typeof(int), typeof(long), typeof(int), typeof(bool));
            int idx = 0;
            Address sender = (Address)objs[idx++];
            int fid = (int)objs[idx++];
            long tl = (long)objs[idx++];
            int nf = (int)objs[idx++];
            bool ic = (bool)objs[idx];
            FragInfo fi = deFragLookup(g, sender, fid, tl, nf, ic);
            if (fi == null)
                return;
            bool doDelivery = false;
            using (new LockAndElevate(fi.Lock))
            {
                if ((IsisSystem.Debug & IsisSystem.FRAGER) != 0)
                    Isis.WriteLine("deFragmenter rdv: senderId=" + Isis.my_address + ", fid=" + fid + (fi == null ? " (** SENDER ON RIP LIST: IGNORING **)" : ""));
                fi.TrueMsg = m;
                if (fi.nFragsRemaining == 0)
                    doDelivery = true;
            }
            if (doDelivery)
                fragDoDelivery(g, fi);
        }

        private static void fragDoDelivery(Group g, FragInfo fi)
        {
            Msg m;
            using (new LockAndElevate(dfLock))
                dfList.Remove(fi);
            if (fi.TrueMsg == null)
                throw new IsisException("Fragger: Rdv failure");
            using (new LockAndElevate(fi.Lock))
            {
                if (fi.iscomp)
                    fi.body = ReliableSender.DeCompress(fi.body);
                m = new Msg(fi.body);
                m.dest = fi.TrueMsg.dest;
                m.flags = (byte)((fi.TrueMsg.flags & ~Msg.FRAGGED) | Msg.DEFRAGGED);
                m.gaddr = fi.TrueMsg.gaddr;
                m.msgid = fi.TrueMsg.msgid;
                m.sender = fi.TrueMsg.sender;
                m.Lid = fi.TrueMsg.Lid;
                // Don't use the reassembled message to satisfy retransmission requests
                m.UID = -1;
                m.type = fi.TrueMsg.type;
                m.vid = fi.TrueMsg.vid;
                // The "true message" will be used if responding to a NACK using a message on m.unstable
                m.asReceived = fi.TrueMsg;
                deFragDone(fi);
            }
            m.myObs = null;
            if ((m.flags & Msg.TOKEN) != 0)
                ReliableSender.gotToken(m);
            else if ((m.flags & Msg.HASREPLY) != 0)
                AwaitReplies.gotReply(m);
            else
            {
                if (m.type == Msg.ISGRPP2P || m.type == Msg.ISRAWGRPP2P)
                    g.doAction(m);
                else
                    g.doDeliveryCallbacks(m, "deFragGotFrag", Msg.FIFOCAST);
            }
        }

        internal static void deFragDone(FragInfo fi)
        {
            fi.body = null;
            fi.TrueMsg = null;
        }

        internal static void deFragNoteFailure(Address who)
        {
            using (new LockAndElevate(dfLock))
            {
                List<FragInfo> newList = new List<FragInfo>();
                foreach (FragInfo fi in dfList)
                    if (!fi.sender.Equals(who))
                        newList.Add(fi);
                dfList = newList;
            }
        }

        private static bool reEnteredSIF;

        internal static byte[] SendInFrags(bool p2p, Address dest, Group g, byte[] buffer, byte[] bufferAsGiven)
        {
            int fid;
            bool ic = (bufferAsGiven != null);
            using (new LockAndElevate(sendInFragsLock))
            {
                if (reEnteredSIF)
                    throw new IsisException("Recursive entry to SendInFrags");
                reEnteredSIF = true;
                fid = ++nextFid;
                long bl = buffer.Length;
                int off = 0;
                int nf = (int)((bl + Isis.ISIS_FRAGLEN - 1) / Isis.ISIS_FRAGLEN);
                long wbl = bl;
                for (int n = 0; n < nf; n++)
                {
                    byte[] frag = new byte[Math.Min(wbl, Isis.ISIS_FRAGLEN)];
                    Array.Copy(buffer, off, frag, 0, frag.Length);
                    wbl -= frag.Length;
                    off += frag.Length;
                    if (g != null)
                    {
                        // These next lines can't run through the flow-controlled version of P2PSend and Send because of the risk of a deadlock
                        // The core issue is that SendInFrags is currently a static method, since I use it both within groups and also for pure
                        // Isis-Isis communication outside of groups (for example, retransmission of a message on the unstable list).  Thus if we allow
                        // flow control to lock send A, it may be that for the FC state to drain, message B needs to be retransmitted, and for this, fragmented
                        if (p2p)
                            g.doP2PSend(dest, true, Isis.FRAGMENT, Isis.my_address, fid, (long)buffer.Length, nf, n, ic, frag);
                        else
                            g.doSend((g == Isis.ORACLE && !dest.Equals(Isis.ORACLE.gaddr)), false, Isis.FRAGMENT, Isis.my_address, fid, (long)buffer.Length, nf, n, ic, frag);
                    }
                    else
                        throw new IsisException("SendInFragments: can't fragment a p2p non-group message");
                }
                if ((IsisSystem.Debug & IsisSystem.FRAGER) != 0)
                    Isis.WriteLine("Fragmented a " + bl + " byte object and sent it as " + nf + " fragments using senderId=" + Isis.my_address + ", fid=" + fid);
                // Surgery: replace the old message with a reference that will let us match it with the fragmented message
                Msg outer = new Msg(bufferAsGiven == null ? buffer : bufferAsGiven);
                Msg inner = Msg.InnerMsg(outer.payload);
                if (inner == null)
                {
                    // Special for fragmented tokens
                    inner = new Msg(Isis.my_address, fid, bl, nf, ic);
                    inner.dest = g.gaddr;
                }
                else
                {
                    // Normal case: everything except tokens
                    inner.payload = Msg.toBArray(Isis.my_address, fid, bl, nf, ic);
                    inner.cipherPayload = null;
                    inner.flags &= ~Msg.CIPHER & 0xFF;
                }
                outer.payload = Msg.toBArray(inner);
                outer.cipherPayload = null;
                outer.flags &= ~Msg.CIPHER & 0xFF;
                outer.myObs = inner.myObs = null;
                reEnteredSIF = false;
                return outer.toBArray();
            }
        }

        internal byte[] cipherBuf(byte[] buffer)
        {
            if (myAes == null)
                throw new IsisException("<" + gname + ">: ciperBuf but myAES=null");
            if (buffer.Length == 0)
                throw new IsisException("Buffer null in cipherBuf");
            if (buffer.Length == 0)
                return buffer;
            return encipher(buffer);
        }

        internal byte[] CryptoWrap(byte[] buffer)
        {
            return Msg.toBArray(Isis.CRYPTOWRAPPED, cipherBuf(buffer));
        }

        internal void cipherMsg(Msg m)
        {
            using (new LockAndElevate(m.Lock))
            {
                if (m.vid < 0 || m.msgid < 0 || myAes == null || m.cipherPayload != null)
                    return;
                m.myObs = null;
                using (new LockAndElevate(myAesLock))
                    m.cipherPayload = encipher(m.payload);
                m.flags |= Msg.CIPHER;
            }
        }

        internal byte[] decipherBuf(byte[] buffer)
        {
            if (myAes == null)
                throw new IsisException("<" + gname + ">: deciperBuf but myAES=null");
            if (buffer.Length == 0)
                return buffer;
            return decipher(buffer);
        }

        internal void decipherMsg(Msg m)
        {
            if ((m.payload != null && m.cipherPayload != null) || (m.flags & Msg.CIPHER) == 0)
                return;
            using (new LockAndElevate(m.Lock))
            {
                m.myObs = null;
                if (m.cipherPayload == null && m.payload != null)
                    m.cipherPayload = m.payload;
                using (new LockAndElevate(myAesLock))
                    m.payload = decipher(m.cipherPayload);
            }
        }

        private byte[] encipher(byte[] buffer)
        {
            int nb = myAes.BlockSize >> 3;
            byte[] IV = new byte[nb];
            AesSeed.NextBytes(IV);
            return encipher(this, myAes, myAesLock, IV, buffer);
        }

        internal static byte[] encipher(Group g, Aes myAes, LockObject myAesLock, byte[] IV, byte[] buffer)
        {
            using (new LockAndElevate(myAesLock))
            {
                int nb = myAes.BlockSize >> 3;
                myAes.IV = IV;
                if ((IsisSystem.Debug & IsisSystem.CIPHER) != 0)
                {
                    tokenInfo.dumpBv((g != null ? ("<" + g.gname + "> ") : "<null> ") + "[encipher]myAes.Key=", myAes.Key);
                    tokenInfo.dumpBv((g != null ? ("<" + g.gname + "> ") : "<null> ") + "[encipher]myAes.IV=", myAes.IV);
                    Isis.WriteLine((g != null ? ("<" + g.gname + "> ") : "<null> ") + "[encipher]Buffer length is " + buffer.Length);
                }
                int len = (((buffer.Length + 4) + (nb - 1)) / nb) * nb;
                byte[] Padded = new byte[len];
                int idx = 0;
                Padded[idx++] = (byte)((buffer.Length >> 24) & 0xFF);
                Padded[idx++] = (byte)((buffer.Length >> 16) & 0xFF);
                Padded[idx++] = (byte)((buffer.Length >> 8) & 0xFF);
                Padded[idx++] = (byte)(buffer.Length & 0xFF);
                while (idx - 4 < buffer.Length)
                {
                    Padded[idx] = buffer[idx - 4];
                    idx++;
                }
                while (idx < len - 4)
                    Padded[idx++] = 0;
                ICryptoTransform myEncryptor;
                using (myEncryptor = myAes.CreateEncryptor(myAes.Key, myAes.IV))
                using (MemoryStream msEncrypt = new MemoryStream())
                {
                    using (CryptoStream csEncrypt = new CryptoStream(msEncrypt, myEncryptor, CryptoStreamMode.Write))
                        csEncrypt.Write(Padded, 0, Padded.Length);
                    byte[] EV = msEncrypt.ToArray();
                    byte[] result = new byte[IV.Length + EV.Length];
                    Array.Copy(IV, result, IV.Length);
                    Array.Copy(EV, 0, result, IV.Length, EV.Length);
                    return result;
                }
            }
        }

        // Caller has a lock on myAes
        private byte[] decipher(byte[] buffer)
        {
            using (new LockAndElevate(myAesLock))
            {
                int nb = myAes.BlockSize >> 3;
                byte[] result;
                byte[] IV = new byte[nb];
                Array.Copy(buffer, IV, nb);
                myAes.IV = IV;
                if ((IsisSystem.Debug & IsisSystem.CIPHER) != 0)
                {
                    tokenInfo.dumpBv("<" + gname + "> [decipher]myAes.Key=", myAes.Key);
                    tokenInfo.dumpBv("<" + gname + "> [decipher]myAes.IV=", myAes.IV);
                }
                using (myDecryptor = myAes.CreateDecryptor(myAes.Key, myAes.IV))
                using (MemoryStream msDecrypt = new MemoryStream(buffer, nb, buffer.Length - nb))
                {
                    byte[] Padded = new byte[myAes.BlockSize >> 3];
                    using (CryptoStream csDecrypt = new CryptoStream(msDecrypt, myDecryptor, CryptoStreamMode.Read))
                    {
                        csDecrypt.Read(Padded, 0, Padded.Length);
                        int len = ((int)Padded[0] << 24) + ((int)Padded[1] << 16) + ((int)Padded[2] << 8) + ((int)Padded[3]);
                        if (len < 0 || len > Isis.ISIS_MAXMSGLENTOTAL)
                        {
                            Isis.WriteLine("WARNING: Decryption failure (object may be corrupted, or may have been enciphered with a different key)");
                            return new byte[0];
                        }
                        if ((IsisSystem.Debug & IsisSystem.CIPHER) != 0)
                            Isis.WriteLine("[decipher]Length will be " + len);
                        result = new byte[len];
                        int idx = 4, off = 0;
                        while (len-- > 0)
                        {
                            if (idx == Padded.Length)
                                csDecrypt.Read(Padded, idx = 0, Padded.Length);
                            result[off++] = Padded[idx++];
                        }
                    }
                    return result;
                }
            }
        }

        /// <summary>
        /// Sends one message in a stream of messages that constitute a state transfer or checkpoint.
        /// </summary>
        /// <param name="obs">A variable-length list of arguments that must match one of the checkpoint loading methods</param>
        /// <remarks>
        /// Sends one message in a stream of messages that constitute a state transfer or checkpoint.
        /// Each of these messages can contain any objects that the Isis Msg layer is able to Marshall,
        /// namely predefined types known to Isis, or other object types that the user has registered via <see cref="Msg.RegisterType"/>.
        /// The system matches each incoming checkpoint message with the list of registered handlers, invoking the handler(s) that have
        /// exact matches with the types of the incoming message. 
        /// </remarks>
        public void SendChkpt(params object[] obs)
        {
            if (!IsisCallStart())
                return;
            object[] args = new object[obs.Length + 1];
            args[0] = Isis.STATEXFER;
            for (int i = 0; i < obs.Length; i++)
                args[i + 1] = obs[i];
            if (myChkptStream == null)
                foreach (Address a in nextView.joiners)
                {
                    bool sendToHim = false;
                    if (theChkptChoser == null)
                        sendToHim = true;
                    else
                        sendToHim = theChkptChoser(nextView, a);
                    if (sendToHim)
                        doP2PSend(a, true, args);
                }
            else
            {
                try
                {
                    if (obs.Length > 0)
                    {
                        byte[] buffer = Msg.toBArray(obs);
                        if (userSpecifiedKey)
                            cipherBuf(buffer);
                        int len = buffer.Length;
                        byte[] lb = new byte[4];
                        lb[0] = (byte)(len & 0xFF); len >>= 8;
                        lb[1] = (byte)(len & 0xFF); len >>= 8;
                        lb[2] = (byte)(len & 0xFF); len >>= 8;
                        lb[3] = (byte)(len & 0xFF); len >>= 8;
                        myChkptStream.Write(lb, 0, 4);
                        myChkptStream.Write(buffer, 0, buffer.Length);
                    }
                    else
                    {
                        makingCheckpoint = false;
                        myChkptStream.Close();
                        myChkptStream = null;
                        if (!File.Exists(myCheckpointFile + ".chkpt"))
                            File.Create(myCheckpointFile + ".chkpt").Close();
                        if (!File.Exists(myCheckpointFile + ".bak"))
                            File.Create(myCheckpointFile + ".bak").Close();
                        bool replaced = false;
                        int tried = 0;
                        while (!replaced && tried++ < 3)
                            try
                            {
                                File.Replace(myCheckpointFile + ".tmp", myCheckpointFile + ".chkpt", myCheckpointFile + ".bak", false);
                                replaced = true;
                            }
                            catch (IOException)
                            {
                                if (tried == 3)
                                    throw;
                                Thread.Sleep(250);
                                continue;
                            }
                    }
                }
                catch (Exception e)
                {
                    throw new IsisException("I/O error: " + e + " while writing checkpoint");
                }
            }
            IsisCallDone();
        }

        /// <summary>
        /// Called as an end-of-state-transfer or end-of-checkpoint marker.
        /// </summary>
        public void EndOfChkpt()
        {
            if (!IsisCallStart())
                return;
            if (!inhibitEOC)
                SendChkpt();
            IsisCallDone();
        }

        private static string bprnt(byte[] buf)
        {
            string s = "";
            for (int b = 0; b < buf.Length; b++) s = s + " " + buf[b];
            return s;
        }

        /// <summary>
        /// Issues an unordered, non-durable point-to-point request to a designated member of some group and then waits for it to 
        /// reply.
        /// </summary>
        /// <param name="dest">the target for the query</param>
        /// <param name="obs">A variable list of arguments that specify the request id and the arguments to some handler</param>
        /// <returns>Return returned as a byte[] array</returns>
        /// <remarks>
        /// Issues an unordered, non-durable point-to-point request to a designated member of some group and then waits for it to 
        /// reply (<see cref="Reply"/>, <see cref="NullReply"/> and <see cref="AbortReply"/>).
        /// The first parameter is an object of type <see cref="Timeout"/> and specifies a timeout after which the Query ceases to wait for a non-responsive
        /// member, and the default action to take in that case.  The next parameter is the request handle: a small integer identifying this request.
        /// Remaining parameters become typed arguments to the handler for the request.  Returns a byte[] array in which the reply is encoded.
        /// Normally, the user would employ Msg.BArraysToObjects() or Msg.InvokeFromBArrays() to decode these replies.
        /// </remarks>
        public byte[] P2PQueryToBarray(Address dest, params object[] obs)
        {
            if (!IsisCallStart())
                return null;
            Timeout timeout;
            splitObs(this, out timeout, ref obs);
            cbCheck(obs);
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            byte[] buffer = Msg.toBArray(obs);
            if ((flags & G_SECURE) != 0)
                buffer = CryptoWrap(buffer);
            byte[] rv = ReliableSender.QueryP2P(Msg.ISGRPP2P, dest, timeout, this, buffer);
            IsisCallDone();
            return rv;
        }

        /// <summary>
        /// Issues an unordered, non-durable point-to-point request to a designated member of some group and then waits for it to 
        /// reply.
        /// </summary>
        /// <param name="dest">the target for the query</param>
        /// <param name="obs">A variable list of arguments that specify the request id and the arguments to some handler</param>
        /// <returns>Returns result via user-supplied list objects</returns>
        /// <remarks>
        /// Issues an unordered, non-durable point-to-point request to a designated member of some group and then waits for it to 
        /// reply (<see cref="Reply"/>, <see cref="NullReply"/> and <see cref="AbortReply"/>).
        /// The first parameter is an object of type <see cref="Timeout"/> and specifies a timeout after which the Query ceases to wait for a non-responsive
        /// member, and the default action to take in that case.  The next parameter is the request handle: a small integer identifying this request.
        /// Remaining parameters become typed arguments to the handler for the request.  Returns a byte[] array in which the reply is encoded.
        /// Normally, the user would employ Msg.BArraysToObjects() or Msg.InvokeFromBArrays() to decode these replies.
        /// </remarks>
        public bool P2PQuery(Address dest, params object[] obs)
        {
            if (!IsisCallStart())
                return false;
            FlowControl.FCBarrierCheck();
            Timeout timeout;
            object[] resRefs;
            splitObs(this, out timeout, ref obs, out resRefs);
            cbCheck(obs);
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            byte[] buffer = Msg.toBArray(obs);
            if ((flags & G_SECURE) != 0)
                buffer = CryptoWrap(buffer);
            buffer = ReliableSender.QueryP2P(Msg.ISGRPP2P, dest, timeout, this, buffer);
            List<byte[]> barrays = new List<byte[]>();
            if (buffer == null || buffer.Length == 0)
            {
                IsisCallDone();
                return false;
            }
            barrays.Add(buffer);
            Msg.BArraysToLists(resRefs, barrays);
            IsisCallDone();
            return true;
        }

        internal byte[] doP2PQuery(Address dest, params object[] obs)
        {
            Timeout timeout;
            splitObs(this, out timeout, ref obs);
            cbCheck(obs);
            byte[] buffer = Msg.toBArray(obs);
            if ((flags & G_SECURE) != 0)
                buffer = CryptoWrap(buffer);
            return ReliableSender.QueryP2P(Msg.ISGRPP2P, dest, timeout, this, buffer);
        }


        internal byte[] doPureP2PQuery(Address dest, params object[] obs)
        {
            Timeout timeout;
            splitObs(this, out timeout, ref obs);
            cbCheck(obs);
            byte[] buffer = Msg.toBArray(obs);
            if ((flags & G_SECURE) != 0)
                buffer = CryptoWrap(buffer);
            return ReliableSender.QueryP2P(Msg.ISPUREP2P, dest, timeout, this, buffer);
        }

        /// <summary>
        /// Sends a point-to-point message to a specific member of the current group, specified via the dest field.  
        /// </summary>
        /// <param name="dest">Which member of the group to send to</param>
        /// <param name="obs">A variable list of arguments that specify the request id and the arguments to some handler</param>
        /// <remarks>
        /// Sends a point-to-point message to a specific member of the current group, specified via the dest field.  
        /// The
        /// first parameter specified the request handle: a small integer identifying this request.
        /// Remaining parameters become typed arguments to the handler for the request.
        /// </remarks>
        public void P2PSend(Address dest, params object[] obs)
        {
            if (!IsisCallStart())
                return;
            FlowControl.FCBarrierCheck();
            cbCheck(obs);
            byte[] buffer = Msg.toBArray(obs);
            if ((flags & G_SECURE) != 0)
                buffer = CryptoWrap(buffer);
            ReliableSender.SendP2P(Msg.ISGRPP2P, dest, this, buffer, true);
            IsisCallDone();
        }

        internal void doP2PSend(Address dest, bool localSender, params object[] obs)
        {
            byte[] buffer = Msg.toBArray(obs);
            if ((flags & G_SECURE) != 0)
                buffer = CryptoWrap(buffer);
            ReliableSender.SendP2P(Msg.ISGRPP2P, dest, this, buffer, localSender);
        }

        internal void doPureP2PSend(Address dest, bool localSender, params object[] obs)
        {
            byte[] buffer = Msg.toBArray(obs);
            ReliableSender.SendP2P(Msg.ISPUREP2P, dest, null, buffer, localSender);
        }

        /// <summary>
        /// Used to send an unreliable datagram.  Best if the object size is well below the ISIS_MAXPACKETLEN
        /// </summary>
        /// <param name="dest">Target node within this group</param>
        /// <param name="obs">Request code and parameters</param>
        /// <remarks>
        /// RawP2PSend is used to send an unreliable datagram, for example in support of a gossip-push protocol.  
        /// No attempt will be made to retransmit if the message is dropped.
        /// Caution: if a very large object is sent this way, and some fragments are lost, resources will be tied up
        /// for many seconds on the receiver until the garbage collection logic notices the unrepaired gap and discards
        /// the partial packet.  We recommend that SendRawP2P not be used for objects that exceed the ISIS_MAXPACKETLEN 
        /// limit (in fact, because your data will be encoded and might be enciphered, you need to limit your packets
        /// to a considerably smaller size.  In the worst case many hundreds of bytes of overhead may be added by
        /// these mechanisms, and by the Msg serialization code)
        /// </remarks>
        public void RawP2PSend(Address dest, params object[] obs)
        {
            if (!IsisCallStart())
                return;
            FlowControl.FCBarrierCheck();
            cbCheck(obs);
            byte[] buffer = Msg.toBArray(obs);
            if ((flags & G_SECURE) != 0)
                buffer = CryptoWrap(buffer);
            ReliableSender.SendP2P(Msg.ISRAWGRPP2P, dest, this, buffer, true);
            IsisCallDone();
        }

        /// <summary>
        /// Used to send an unreliable query, to which the receiver will reply.  Best if the object size is well below the ISIS_MAXPACKETLEN
        /// </summary>
        /// <param name="dest">Target node, within this group</param>
        /// <param name="obs">Request code and parameters</param>
        /// <remarks>
        /// RawP2PQuery is used to send an unreliable query, for example in support of a gossip-pull or push-pull protocol.  
        /// No attempt will be made to retransmit if the message is dropped.
        /// Caution: if a very large object is sent this way, and some fragments are lost, resources will be tied up
        /// for many seconds on the receiver until the garbage collection logic notices the unrepaired gap and discards
        /// the partial packet.  We recommend that SendRawP2P not be used for objects that exceed the ISIS_MAXPACKETLEN 
        /// limit (in fact, because your data will be encoded and might be enciphered, you need to limit your packets
        /// to a considerably smaller size.  In the worst case many hundreds of bytes of overhead may be added by
        /// these mechanisms, and by the Msg serialization code)
        /// 
        /// RawP2PQuery is useful in time-sensitive applications where a message that might be delayed and need to be
        /// resent would be of low value because of the elapsed time.  Often, one uses RawReply to respond to such a 
        /// Query, for the same reason.
        /// 
        /// If a timeout occurs, the specified timeout action will be taken
        /// </remarks>
        public bool RawP2PQuery(Address dest, params object[] obs)
        {
            if (!IsisCallStart())
                return false;
            FlowControl.FCBarrierCheck();
            Timeout timeout;
            object[] resRefs;
            splitObs(this, out timeout, ref obs, out resRefs);
            cbCheck(obs);
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            byte[] buffer = Msg.toBArray(obs);
            if ((flags & G_SECURE) != 0)
                buffer = CryptoWrap(buffer);
            buffer = ReliableSender.QueryP2P(Msg.ISRAWGRPP2P, dest, timeout, this, buffer);
            List<byte[]> barrays = new List<byte[]>();
            if (buffer == null || buffer.Length == 0)
            {
                IsisCallDone();
                return false;
            }
            barrays.Add(buffer);
            Msg.BArraysToLists(resRefs, barrays);
            IsisCallDone();
            return true;
        }

        /// <summary>
        /// Used to send an unreliable datagram in reply to a RawP2PQuery.  Best if the reply object size is well below the ISIS_MAXPACKETLEN
        /// </summary>
        /// <param name="obs">The reply data</param>
        /// <remarks>
        /// RawReply is used in conjunction with RawP2PQuery, and sends a reply unreliably: no attempt will be made to retransmit if the message is dropped.
        /// Caution: if a very large object is sent this way, and some fragments are lost, resources will be tied up
        /// for many seconds on the receiver until the garbage collection logic notices the unrepaired gap and discards
        /// the partial packet.  We recommend that SendRawP2P not be used for objects that exceed the ISIS_MAXPACKETLEN 
        /// limit (in fact, because your data will be encoded and might be enciphered, you need to limit your packets
        /// to a considerably smaller size.  In the worst case many hundreds of bytes of overhead may be added by
        /// these mechanisms, and by the Msg serialization code)
        /// 
        /// Note that Reply() also works for RawP2PQuery: the query is sent unreliably but the reply will be acked.  However, this mixture is uncommon.
        /// The converse is also true: nothing prevents an application from using RawReply to respond to a Query sent reliably.
        /// 
        /// If the reply is dropped, the Query or RawQuery will eventually time out and take the indicated timeout action.
        /// </remarks>
        public void RawReply(params object[] obs)
        {
            if (GroupOpen == false && WasOpen)
                return;
            if (!IsisCallStart())
                return;
            Msg replyTo;
            using (new LockAndElevate(Rlock))
                replyTo = getReplyToAndClear();
            if (replyTo == null)
                throw new IsisException("Attempted to reply twice to same message, or to a message that wasn't a query");
            if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                Isis.WriteLine("Sending reply to " + replyTo.sender + ", " + replyTo.vid + ":" + replyTo.msgid);
            byte[] result = Msg.toBArray(obs);
            bool enciphered = false;
            if ((flags & G_SECURE) != 0 && (replyTo.flags & Msg.ENCIPHEREDREPLY) != 0)
            {
                enciphered = true;
                result = cipherBuf(result);
            }
            bool deliverToOracle = (replyTo.flags & Msg.SENTBYORACLE) != 0;
            byte[] buffer = Msg.toBArray(RT_REPLY, replyTo.vid, replyTo.msgid, deliverToOracle, enciphered, result);
            Isis.PendingLeaderOps plos;
            using (new LockAndElevate(groupLock))
                plos = NotifyDALOnReply;
            if (plos != null && plos.reqMsg == replyTo)
                Isis.DALReplyNotify(this, new Msg(gaddr, Msg.ISREPLY, replyTo.sender, Msg.NewMsgAsBArray(Isis.my_address, gaddr, theView.viewid, -1, 0L, 0, 0, 0, buffer), theView.viewid, -1), plos, replyTo);
            ReliableSender.SendP2P(Msg.ISRAWREPLY, replyTo.sender, rgroup(replyTo.gaddr), theView == null ? 0 : theView.viewid, ReliableSender.P2PSequencer.NextP2PSeqn("reply/1", replyTo.sender),
                buffer, true, null, replyTo);
            IsisCallDone();
        }

        /// <exclude>
        /// <summary>
        /// Internal, declared public to satisfy C# scoping requirement.
        /// </summary>
        /// <param name="obs"></param>
        /// </exclude>
        public delegate void querySender(params object[] obs);

        /// <summary>
        /// Issues an unordered, non-durable multicast to the group and waits for replies, returning them as a byte[][] array, with one byte[] vector per reply.
        /// </summary>
        /// <param name="nreplies">number of replies desired</param>
        /// <param name="timeout">Isis.Timeout object specifying the timeout and action to take if it occurs before reply is received</param>
        /// <param name="obs">variable-length list specifying method to invoke, arameters to method being invoked</param>
        /// <returns>a byte[][] array with one byte[] vector per received reply</returns>
        /// <remarks>
        /// Issues an unordered, non-durable multicast to the group and waits for replies (<see cref="Reply"/>, <see cref="NullReply"/> and <see cref="AbortReply"/>).
        /// 
        /// This is the very fastest form of virtually synchronous query, but can be lost in the event of a failure.  Typically performed as a single IP multicast with
        /// instant invocation of the handler routine in the receiving processes.  Read about the <it>virtual synchrony model</it> to learn more about when Query is a safe choice.
        /// 
        /// The number of replies desired can be specified as an integer (normally 1 or 2), or as the special constant ALL.  The first
        /// of the parameter after nreplies is an object of type <see cref="Timeout"/> and specifies a timeout after which the Query ceases to wait for a non-responsive
        /// member, and the default action to take in that case.  The next parameter is the request handle: a small integer identifying this request.
        /// Remaining parameters become typed arguments to the handler for the request.  Returns a byte[][] array in which each entry is a reply from one process.
        /// Normally, the user would employ Msg.BArraysToObjects() or Msg.InvokeFromBArrays() to decode these replies.
        /// </remarks>
        public List<byte[]> QueryToBA(int nreplies, Timeout timeout, params object[] obs)
        {
            return _Query(false, false, (querySender)doSendNotFromOracle, nreplies, timeout, obs);
        }

        /// <summary>
        /// Unreliable version of QueryToBA
        /// </summary>
        /// <param name="nreplies">number of replies desired</param>
        /// <param name="timeout">timeout action</param>
        /// <param name="obs">arguments to the call</param>
        /// <returns>byte[][] array containing the replies</returns>
        public List<byte[]> RawQueryToBA(int nreplies, Timeout timeout, params object[] obs)
        {
            return _Query(false, true, (querySender)doRawSendNotFromOracle, nreplies, timeout, obs);
        }

        private List<byte[]> QueryToBAFromSystem(int vid, int mid, int nRaw, int nreplies, Timeout timeout, params object[] obs)
        {
            return _Query(vid, mid, nRaw, false, false, (querySender)doSendNotFromOracle, nreplies, timeout, obs);
        }

        /// <summary>
        ///  Issues an unordered and reliable but potentially non-durable multicast to the group, then waits for replies
        /// </summary>
        /// <param name="nreplies">number of replies desired</param>
        /// <param name="given">variable-length list specifying method to invoke, timeout, parameters to method being invoked, EOLMarker, vectors for received results</param>
        /// <returns>how many replies were actually received</returns>
        /// <remarks>
        /// Issues an unordered and reliable but potentially non-durable multicast to the group and waits for replies (<see cref="Reply"/>, <see cref="NullReply"/> and <see cref="AbortReply"/>).
        /// 
        /// This is the very fastest way to query an Isis group; it typically requires a single IP multicast per invocation.  However, there are obscure failure patterns that
        /// could cause it to be nondurable (the query is "lost" in the event of certain sequences of failures).  
        /// Read about the <it>virtual synchrony model</it> to learn more about when stronger Query variations would be needed.
        /// 
        /// The number of replies desired can be specified as an integer (normally 1 or 2), or as the special constant ALL.  The first
        /// of the parameter after nreplies is an object of type <see cref="Timeout"/> and specifies a timeout after which the Query ceases to wait for a non-responsive
        /// member, and the default action to take in that case.  The next parameter is the request handle: a small integer identifying this request.
        /// Remaining parameters become typed arguments to the handler for the request.  Counts the replies received and returns this number but the actual replies
        /// are passed to the user in a series of vectors, which should be passed in as by-ref parameters after a marker, the Isis EOLmarker, which separates
        /// the list of arguments to the invoked method from the places to put recieved replies.
        /// 
        /// For example:  int[] hisAge = new int[0], nreps;  nreps = Query(GET_AGE, 1, new Timeout(1000, TO_ABORTREPLY), "John Smith", Isis.EOLmarker, hisAge);
        /// </remarks>
        public int Query(int nreplies, params object[] given)
        {
            Timeout timeout;
            object[] resRefs;
            splitObs(this, out timeout, ref given, out resRefs);
            List<byte[]> barrays = QueryToBA(nreplies, timeout, given);
            Msg.BArraysToLists(resRefs, barrays);
            return barrays.Count();
        }

        /// <summary>
        /// Like Query, but uses RawSend
        /// </summary>
        /// <param name="nreplies">number of replies desired</param>
        /// <param name="given">variable-length list specifying method to invoke, timeout, parameters to method being invoked, EOLMarker, vectors for received resultsvariable-length list specifying method to invoke, timeout, parameters to method being invoked, EOLMarker, vectors for received results</param>
        /// <returns></returns>
        public int RawQuery(int nreplies, params object[] given)
        {
            Timeout timeout;
            object[] resRefs;
            splitObs(this, out timeout, ref given, out resRefs);
            List<byte[]> barrays = RawQueryToBA(nreplies, timeout, given);
            Msg.BArraysToLists(resRefs, barrays);
            return barrays.Count();
        }

        internal int doQuery(int nreplies, params object[] given)
        {
            Timeout timeout;
            object[] resRefs;
            splitObs(this, out timeout, ref given, out resRefs);
            List<byte[]> barrays = doQueryToBA(false, nreplies, timeout, given);
            Msg.BArraysToLists(resRefs, barrays);
            return barrays.Count();
        }

        internal static void splitObs(object g, out Timeout timeout, ref object[] obs, out object[] refs)
        {
            splitObs(g, out timeout, ref obs);
            int nRefs = 0;
            while (nRefs < obs.Length && !obs[(obs.Length - 1) - nRefs].GetType().Equals(typeof(EOLMarker)))
                ++nRefs;
            if (nRefs == obs.Length)
            {
                refs = new object[0];
                return;
            }
            refs = new object[nRefs];
            for (int i = 0; i < nRefs; i++)
            {
                try
                {
                    refs[i] = obs[obs.Length - nRefs + i];
                    if (refs[i] == null)
                        throw new ArgumentException();
                    int len = 0;
                    foreach (object o in (IList)refs[i])
                        ++len;
                    // At this point we used to complain if the length wasn't initially 0, but these days we allow
                    // "Accumulators" hence the exception is no longer thrown.  But the logic above still is useful:
                    // an exception will be thrown if something is wrong (better to throw it when the call is first made rather than later..).
                }
                catch
                {
                    throw new ArgumentException("In an Isis Query, all reply variables must be lists");
                }
            }
            Isis.ArrayResize<object>(ref obs, obs.Length - nRefs - 1);
        }

        internal static void splitObs(object g, out Timeout timeout, ref object[] obs)
        {
            checkGroupIsOpen(g);
            if (obs[0].GetType().Equals(typeof(Timeout)))
            {
                timeout = (Timeout)obs[0];
                for (int i = 0; i < obs.Length - 1; i++)
                    obs[i] = obs[i + 1];
                Isis.ArrayResize<object>(ref obs, obs.Length - 1);
            }
            else
                timeout = new Timeout(Isis.ISIS_DEFAULTTIMEOUT, Timeout.TO_ABORTREPLY);
            obs = subsetShortcut(obs);
        }

        internal void QueryInvoke(int nreplies, params object[] obs)
        {
            Timeout timeout;
            splitObs(this, out timeout, ref obs);
            Delegate del = (Delegate)obs[obs.Length - 1];
            Isis.ArrayResize<object>(ref obs, obs.Length - 1);
            Msg.InvokeFromBArrays(QueryToBA(nreplies, timeout, obs), del);
        }

        internal List<byte[]> doQueryToBA(bool sentByOracle, int nreplies, params object[] obs)
        {
            Timeout timeout;
            splitObs(this, out timeout, ref obs);
            return _Query(sentByOracle, false, sentByOracle ? (querySender)doSendFromOracle : (querySender)doSendNotFromOracle, nreplies, timeout, obs);
        }

        internal void doQueryInvoke(int nreplies, params object[] obs)
        {
            Timeout timeout;
            splitObs(this, out timeout, ref obs);
            Delegate del = (Delegate)obs[obs.Length - 1];
            Isis.ArrayResize<object>(ref obs, obs.Length - 1);
            Msg.InvokeFromBArrays(doQueryToBA(false, nreplies, timeout, obs), del);
        }

        /// <summary>
        ///  Issues an ordered but potentially non-durable multicast to the group and waits for replies
        /// </summary>
        /// <param name="nreplies">number of replies desired</param>
        /// <param name="given">variable-length list specifying method to invoke, timeout, parameters to method being invoked, EOLMarker, vectors for received results</param>
        /// <returns>how many replies were actually received</returns>
        /// <remarks>
        /// Issues an ordered but potentially non-durable multicast to the group and waits for replies (<see cref="Reply"/>, <see cref="NullReply"/> and <see cref="AbortReply"/>).
        /// 
        /// This is a slightly slower form of virtually synchronous query, and may not be durable be lost in the event of a failure.  Can be done in a single IP multicast if
        /// the sender was the group leader but if not, delivery will be delayed until the group leader decides and notifies members of the ordering to use.
        /// Read about the <it>virtual synchrony model</it> to learn more about when OrderedQuery is needed.
        /// 
        /// The number of replies desired can be specified as an integer (normally 1 or 2), or as the special constant ALL.  The first
        /// of the parameter after nreplies is an object of type <see cref="Timeout"/> and specifies a timeout after which the Query ceases to wait for a non-responsive
        /// member, and the default action to take in that case.  The next parameter is the request handle: a small integer identifying this request.
        /// Remaining parameters become typed arguments to the handler for the request.  Counts the replies received and returns this number but the actual replies
        /// are passed to the user in a series of vectors, which should be passed in as by-ref parameters after a marker, the Isis EOLmarker, which separates
        /// the list of arguments to the invoked method from the places to put recieved replies.
        /// 
        /// For example:  int[] hisAge = new int[0], nreps;  nreps = OrderedQuery(GET_AGE, 1, new Timeout(1000, TO_ABORTREPLY), "John Smith", Isis.EOLmarker, hisAge);
        /// </remarks>
        public int OrderedQuery(int nreplies, params object[] given)
        {
            Timeout timeout;
            object[] resRefs;
            splitObs(this, out timeout, ref given, out resRefs);
            List<byte[]> barrays = OrderedQueryToBA(nreplies, timeout, given);
            Msg.BArraysToLists(resRefs, barrays);
            return barrays.Count();
        }

        internal int doOrderedQuery(int nreplies, params object[] given)
        {
            Timeout timeout;
            object[] resRefs;
            splitObs(this, out timeout, ref given, out resRefs);
            List<byte[]> barrays = doOrderedQueryToBA(nreplies, timeout, given);
            Msg.BArraysToLists(resRefs, barrays);
            return barrays.Count();
        }

        /// <summary>
        /// Issues a totally ordered (but not necessarily durable) multicast to the group and waits for replies.
        /// </summary>
        /// <param name="nreplies">number of replies desired</param>
        /// <param name="obs">variable-length list specifying method to invoke, timeout, parameters to method being invoked</param>
        /// <returns>a List contaiining one byte[] vector per received reply</returns>
        /// <remarks>
        /// Issues a totally ordered (but not necessarily durable) multicast to the group and waits for replies (<see cref="Reply"/>, <see cref="NullReply"/> and <see cref="AbortReply"/>).
        /// The number of replies desired can be specified as an integer (normally 1 or 2), or as the special constant ALL.  The first
        /// of the parameter after nreplies is an object of type <see cref="Timeout"/> and specifies a timeout after which the Query ceases to wait for a non-responsive
        /// member, and the default action to take in that case.  The next parameter is the request handle: a small integer identifying this request.
        /// Remaining parameters become typed arguments to the handler for the request.  Returns a byte[][] array in which each entry is a reply from one process.
        /// Normally, the user would employ Msg.BArraysToObjects() or Msg.InvokeFromBArrays() to decode these replies.
        /// 
        /// This is a slightly slower form of virtually synchronous query, and may not be durable be lost in the event of a failure.  Can be done in a single IP multicast if
        /// the sender was the group leader but if not, delivery will be delayed until the group leader decides and notifies members of the ordering to use.
        /// Read about the <it>virtual synchrony model</it> to learn more about when OrderedQuery is needed.
        /// </remarks>
        public List<byte[]> OrderedQueryToBA(int nreplies, params object[] obs)
        {
            Timeout timeout;
            splitObs(this, out timeout, ref obs);
            return _Query(false, false, (querySender)OrderedSend, nreplies, timeout, obs);
        }

        /// <summary>
        ///  Issues a causally ordered but potentially non-durable multicast to the group and waits for replies
        /// </summary>
        /// <param name="nreplies">number of replies desired</param>
        /// <param name="given">variable-length list specifying method to invoke, timeout, parameters to method being invoked, EOLMarker, vectors for received results</param>
        /// <returns>how many replies were actually received</returns>
        /// <remarks>
        /// Issues  a causally ordered but potentially non-durable multicast to the group and waits for replies (<see cref="Reply"/>, <see cref="NullReply"/> and <see cref="AbortReply"/>).
        /// 
        /// This is a slightly slower form of virtually synchronous query, and may not be durable be lost in the event of a failure.  Can be done in a single IP multicast 
        /// but delivery might be slightly delayed if the causal ordering would otherwise be violated.
        /// Read about the <it>virtual synchrony model</it> to learn more about when CausalQuery is needed.
        /// 
        /// The number of replies desired can be specified as an integer (normally 1 or 2), or as the special constant ALL.  The first
        /// of the parameter after nreplies is an object of type <see cref="Timeout"/> and specifies a timeout after which the Query ceases to wait for a non-responsive
        /// member, and the default action to take in that case.  The next parameter is the request handle: a small integer identifying this request.
        /// Remaining parameters become typed arguments to the handler for the request.  Counts the replies received and returns this number but the actual replies
        /// are passed to the user in a series of vectors, which should be passed in as by-ref parameters after a marker, the Isis EOLmarker, which separates
        /// the list of arguments to the invoked method from the places to put recieved replies.
        /// 
        /// For example:  int[] hisAge = new int[0], nreps;  nreps = CausalQuery(GET_AGE, 1, new Timeout(1000, TO_ABORTREPLY), "John Smith", Isis.EOLmarker, hisAge);
        /// </remarks>
        public int CausalQuery(int nreplies, params object[] given)
        {
            Timeout timeout;
            object[] resRefs;
            splitObs(this, out timeout, ref given, out resRefs);
            List<byte[]> barrays = CausalQueryToBA(nreplies, timeout, given);
            Msg.BArraysToLists(resRefs, barrays);
            return barrays.Count();
        }

        /// <summary>
        /// Issues a causally ordered (but not necessarily durable) multicast to the group and waits for replies.
        /// </summary>
        /// <param name="nreplies">number of replies desired</param>
        /// <param name="obs">variable-length list specifying method to invoke, timeout, parameters to method being invoked</param>
        /// <returns>a List contaiining one byte[] vector per received reply</returns>
        /// <remarks>
        /// Issues a causally ordered (but not necessarily durable) multicast to the group and waits for replies (<see cref="Reply"/>, <see cref="NullReply"/> and <see cref="AbortReply"/>).
        /// The number of replies desired can be specified as an integer (normally 1 or 2), or as the special constant ALL.  The first
        /// of the parameter after nreplies is an object of type <see cref="Timeout"/> and specifies a timeout after which the Query ceases to wait for a non-responsive
        /// member, and the default action to take in that case.  The next parameter is the request handle: a small integer identifying this request.
        /// Remaining parameters become typed arguments to the handler for the request.  Returns a byte[][] array in which each entry is a reply from one process.
        /// Normally, the user would employ Msg.BArraysToObjects() or Msg.InvokeFromBArrays() to decode these replies.
        /// 
        /// This is a slightly slower form of virtually synchronous query, and may not be durable be lost in the event of a failure.  Can be done in a single IP multicast 
        /// but delivery might be slightly delayed if the causal ordering would otherwise be violated.
        /// Read about the <it>virtual synchrony model</it> to learn more about when CausalQuery is needed.
        /// </remarks>
        public List<byte[]> CausalQueryToBA(int nreplies, params object[] obs)
        {
            Timeout timeout;
            splitObs(this, out timeout, ref obs);
            return _Query(false, false, (querySender)CausalSend, nreplies, timeout, obs);
        }

        internal List<byte[]> doOrderedQueryToBA(int nreplies, params object[] obs)
        {
            Timeout timeout;
            splitObs(this, out timeout, ref obs);
            return _Query(false, false, (querySender)OrderedSend, nreplies, timeout, obs);
        }

        internal void doOrderedQueryInvoke(int nreplies, params object[] obs)
        {
            Timeout timeout;
            splitObs(this, out timeout, ref obs);
            Delegate del = (Delegate)obs[obs.Length - 1];
            Isis.ArrayResize<object>(ref obs, obs.Length - 1);
            Msg.InvokeFromBArrays(doOrderedQueryToBA(nreplies, timeout, obs), del);
        }

        /// <summary>
        /// Issues a Paxos-style (ordered, durable) multicast to the group and returns a byte[] vector encoding replies.
        /// </summary>
        /// <param name="nreplies">number of replies desired</param>
        /// <param name="obs">variable-length list specifying method to invoke, timeout, parameters to method being invoked</param>
        /// <returns>a List containing one byte[] vector per received reply</returns>
        /// <remarks>
        /// Issues a Paxos-style (ordered, durable) multicast to the group and returns a byte[] vector encoding replies.
        /// waits for replies (<see cref="Reply"/>, <see cref="NullReply"/> and <see cref="AbortReply"/>).
        /// 
        /// This is the slowest but most robust form of virtually synchronous query, and matches the State Machine Replication model.  Requires a form of internal 2-phase commit, which
        /// will involve round-trip acks from a majority of group members.  Read about the <it>virtual synchrony model</it> to learn more about when SafeQuery is needed.
        /// 
        /// The number of replies desired can be specified as an integer (normally 1 or 2), or as the special constant ALL.  The first
        /// of the parameter after nreplies is an object of type <see cref="Timeout"/> and specifies a timeout after which the Query ceases to wait for a non-responsive
        /// member, and the default action to take in that case.  The next parameter is the request handle: a small integer identifying this request.
        /// Remaining parameters become typed arguments to the handler for the request.  Returns a List in which each entry is a reply from one process.
        /// Normally, the user would employ Msg.BArraysToObjects() or Msg.InvokeFromBArrays() to decode these replies.
        /// </remarks>
        public List<byte[]> SafeQueryToBA(int nreplies, params object[] obs)
        {
            Timeout timeout;
            splitObs(this, out timeout, ref obs);
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            return _Query(false, false, (querySender)SafeSend, nreplies, timeout, obs);
        }

        /// <summary>
        /// Issues a Paxos-style (ordered, durable) multicast to the group and waits for replies
        /// </summary>
        /// <param name="nreplies">number of replies desired, or MAJORITY, or ALL</param>
        /// <param name="given">variable-length list specifying method to invoke, timeout, parameters, EOLMarker, and result vectors</param>
        /// <returns>number of replies actually received</returns>
        /// <remarks>
        /// Issues a Paxos-style (ordered, durable) multicast to the group and waits for replies (<see cref="Reply"/>, <see cref="NullReply"/> and <see cref="AbortReply"/>).
        /// 
        /// This is the slowest but most robust form of virtually synchronous query, and matches the State Machine Replication model.  Requires a form of internal 2-phase commit, which
        /// will involve round-trip acks from a majority of group members.  Read about the <it>virtual synchrony model</it> to learn more about when SafeQuery is needed.
        /// 
        /// The number of replies desired can be specified as an integer (normally 1 or 2), or as the special constant ALL.  The first
        /// of the parameter after nreplies is an object of type <see cref="Timeout"/> and specifies a timeout after which the Query ceases to wait for a non-responsive
        /// member, and the default action to take in that case.  The next parameter is the request handle: a small integer identifying this request.
        /// Remaining parameters become typed arguments to the handler for the request.  Returns the number of replies received but discards their contents.
        /// 
        /// For example:  int[] hisAge = new int[0], nreps;  nreps = SafeQuery(GET_AGE, 1, new Timeout(1000, TO_ABORTREPLY), "John Smith", Isis.EOLmarker, hisAge);
        /// </remarks>
        public int SafeQuery(int nreplies, params object[] given)
        {
            Timeout timeout;
            object[] resRefs;
            splitObs(this, out timeout, ref given, out resRefs);
            List<byte[]> barrays = SafeQueryToBA(nreplies, timeout, given);
            Msg.BArraysToLists(resRefs, barrays);
            return barrays.Count();
        }

        internal int doSafeQuery(int nreplies, params object[] given)
        {
            Timeout timeout;
            object[] resRefs;
            splitObs(this, out timeout, ref given, out resRefs);
            List<byte[]> barrays = doSafeQueryToBA(nreplies, timeout, given);
            Msg.BArraysToLists(resRefs, barrays);
            return barrays.Count();
        }

        internal void SafeQueryInvoke(int nreplies, params object[] obs)
        {
            Timeout timeout;
            splitObs(this, out timeout, ref obs);
            Delegate del = (Delegate)obs[obs.Length - 1];
            Isis.ArrayResize<object>(ref obs, obs.Length - 1);
            Msg.InvokeFromBArrays(SafeQueryToBA(nreplies, timeout, obs), del);
        }

        internal List<byte[]> doSafeQueryToBA(int nreplies, params object[] obs)
        {
            Timeout timeout;
            splitObs(this, out timeout, ref obs);
            return _Query(false, false, (querySender)SafeSend, nreplies, timeout, obs);
        }

        internal void doSafeQueryInvoke(int nreplies, params object[] obs)
        {
            Timeout timeout;
            splitObs(this, out timeout, ref obs);
            Delegate del = (Delegate)obs[obs.Length - 1];
            Isis.ArrayResize<object>(ref obs, obs.Length - 1);
            Msg.InvokeFromBArrays(doSafeQueryToBA(nreplies, timeout, obs), del);
        }

        internal List<byte[]> doUnorderedQueryToBA(int nreplies, params object[] obs)
        {
            Timeout timeout;
            splitObs(this, out timeout, ref obs);
            return _Query(false, false, (querySender)doUnorderedSend, nreplies, timeout, obs);
        }

        internal List<byte[]> UnorderedQueryToBA(int nreplies, params object[] obs)
        {
            Timeout timeout;
            splitObs(this, out timeout, ref obs);
            return _Query(false, false, (querySender)doUnorderedSend, nreplies, timeout, obs);
        }

        /// <summary>
        /// Issues an unordered multicast to the group and waits for replies
        /// </summary>
        /// <param name="nreplies">number of replies desired, or MAJORITY, or ALL</param>
        /// <param name="given">variable-length list specifying method to invoke, timeout, parameters, EOLMarker, and result vectors</param>
        /// <returns>number of replies actually received</returns>
        /// <remarks>
        /// Issues an unordered multicast to the group and waits for replies (<see cref="Reply"/>, <see cref="NullReply"/> and <see cref="AbortReply"/>).
        /// The number of replies desired can be specified as an integer (normally 1 or 2), or as the special constant ALL.  The first
        /// of the parameter after nreplies is an object of type <see cref="Timeout"/> and specifies a timeout after which the Query ceases to wait for a non-responsive
        /// member, and the default action to take in that case.  The next parameter is the request handle: a small integer identifying this request.
        /// Remaining parameters become typed arguments to the handler for the request.  Returns the number of replies received but discards their contents.
        /// </remarks>
        public int UnorderedQuery(int nreplies, params object[] given)
        {
            Timeout timeout;
            object[] resRefs;
            splitObs(this, out timeout, ref given, out resRefs);
            List<byte[]> barrays = UnorderedQueryToBA(nreplies, timeout, given);
            Msg.BArraysToLists(resRefs, barrays);
            return barrays.Count();
        }

        internal int doUnorderedQuery(int nreplies, params object[] given)
        {
            Timeout timeout;
            object[] resRefs;
            splitObs(this, out timeout, ref given, out resRefs);
            List<byte[]> barrays = doUnorderedQueryToBA(nreplies, timeout, given);
            Msg.BArraysToLists(resRefs, barrays);
            return barrays.Count();
        }

        internal void doUnorderedQueryInvoke(int nreplies, params object[] obs)
        {
            Timeout timeout;
            splitObs(this, out timeout, ref obs);
            Delegate del = (Delegate)obs[obs.Length - 1];
            Isis.ArrayResize<object>(ref obs, obs.Length - 1);
            Msg.InvokeFromBArrays(doUnorderedQueryToBA(nreplies, timeout, obs), del);
        }

        private List<byte[]> _Query(bool sentByOracle, bool isRaw, querySender AsyncSendQuery, int nreplies, Timeout timeout, object[] obs)
        {
            return _Query(Msg.UNINITIALIZED, Msg.UNINITIALIZED, 0, sentByOracle, isRaw, AsyncSendQuery, nreplies, timeout, obs);
        }

        private List<byte[]> _Query(int vid, int mid, int nRaw, bool sentByOracle, bool isRaw, querySender AsyncSendQuery, int nreplies, Timeout timeout, object[] obs)
        {
            if (!IsisCallStart())
                return new List<byte[]>();
            long mylid = 0;
            if (myLoggingFcn != null)
                myLoggingFcn(IL_QUERY, IL_START, Isis.my_address, mylid = newLoggingId(), obs);
            AwaitReplies.ReplyInfo ri;
            if (GroupOpen == false)
            {
                if ((IsisSystem.Debug & (IsisSystem.VIEWWAIT | IsisSystem.REPLYWAIT)) != 0)
                    Isis.WriteLine("Query to a group with GroupOpen==false: return new byte[0][]");
                IsisCallDone();
                return new List<byte[]>();
            }
            if (obs == null || obs.Length < 1)
                throw new ArgumentNullException("Isis.Group.query");

            Msg m;
            if (obs.Length == 1 && obs[0].GetType().Equals(typeof(Msg)))
                m = (Msg)obs[0];
            else
            {
                object dests = null;
                if (obs.Length == 1 && obs[0].GetType().Equals(typeof(object[])) && ((object[])obs[0]).Length > 0 && IsSubsetSend(((object[])obs[0])[0]))
                {
                    dests = ((object[])obs[0])[0];
                    isRaw = true;
                }
                else if (obs.Length > 0 && IsSubsetSend(obs[0]))
                {
                    dests = obs[0];
                    obs = fixObs(obs);
                    isRaw = true;
                }
                cbCheck(obs);
                m = new Msg(obs);
                // Used to pass additional destination information to the doTheSend code that needs it
                if (dests != null)
                {
                    if (dests.GetType().Equals(typeof(List<Address>)))
                        m.destList = (List<Address>)dests;
                    else
                        m.destList = ((QKD)dests).GetDests(this);
                }
            }
            if (vid != Msg.UNINITIALIZED)
            {
                m.vid = vid;
                m.msgid = mid;
                m.nRaw = nRaw;
            }
            else
                SetMsgIds(m, sentByOracle, isRaw);

            m.flags |= Msg.NEEDSREPLY;
            if ((flags & G_SECURE) != 0)
                m.flags |= Msg.ENCIPHEREDREPLY;
            if (sentByOracle)
                m.flags |= Msg.SENTBYORACLE;

            if (nreplies == 0 || !HasFirstView)
            {
                if ((IsisSystem.Debug & (IsisSystem.VIEWWAIT | IsisSystem.REPLYWAIT)) != 0)
                    Isis.WriteLine("Query to group<" + gname + "> with nreplies=" + nreplies + ", and HasFirstView=" + HasFirstView + ": return new byte[0][]");
                IsisCallDone();
                return new List<byte[]>();
            }
            if (nreplies == ALL)
                nreplies = theView.members.Length;
            else if (nreplies == MAJORITY)
                nreplies = (theView.members.Length + 1) / 2;

            nreplies = Math.Min(nreplies, theView.members.Length);

            if ((IsisSystem.Debug & (IsisSystem.REPLYWAIT | IsisSystem.VIEWCHANGE)) != 0)
                Isis.WriteLine("Registering a wait structure... group " + gname + ", wait-id " + m.vid + ":" + m.msgid + ", nreplies " + nreplies);
            ri = AwaitReplies.registerGroupWait(this, m.vid, m.msgid, m.destList, sentByOracle, (flags & G_SECURE) != 0, nreplies, timeout.when);
            if ((IsisSystem.Debug & (IsisSystem.REPLYWAIT | IsisSystem.VIEWCHANGE | IsisSystem.VIEWWAIT)) != 0)
                Isis.WriteLine("Sending my query: group " + gname + " msgid " + m.vid + ":" + m.msgid + ", ri.wanted=" + ri.replies_wanted);
            AsyncSendQuery(m);
            if ((IsisSystem.Debug & (IsisSystem.REPLYWAIT | IsisSystem.VIEWCHANGE | IsisSystem.VIEWWAIT)) != 0)
                Isis.WriteLine("After sending my query: group " + gname + " msgid " + m.vid + ":" + m.msgid + ".... Collecting the replies...");
            AwaitReplies.awaitReplies(ri, this, timeout, m.destList);
            if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                Isis.WriteLine("Collected replies, list contains " + ri.rdvReplies.Count());
            if ((flags & G_SECURE) != 0)
                DecipherReplies(ri);
            if (myLoggingFcn != null)
                myLoggingFcn(IL_QUERY, IL_DONE, Isis.my_address, mylid);
            IsisCallDone();
            return ri.rdvReplies;
        }

        internal void DecipherReplies(AwaitReplies.ReplyInfo ri)
        {
            List<byte[]> tmp = new List<byte[]>();
            foreach (byte[] r in ri.rdvReplies)
                tmp.Add(decipherBuf(r));
            ri.rdvReplies = tmp;
        }

        internal class querierArgs
        {
            internal List<byte[]>[] ba;
            internal List<byte[]> bap2p;
            internal Group g;
            internal int nr;
            internal bool sentByOracle;
            internal int whoAmI;
            internal Timeout timeout;
            internal object[] obs;

            internal querierArgs(List<byte[]>[] b, Group group, int n, bool bo, int w, Timeout to, object[] o) { ba = b; g = group; nr = n; sentByOracle = bo; whoAmI = w; timeout = to; obs = o; }
            internal querierArgs(List<byte[]> bp2p, Group group, int w, Timeout to, object[] o) { ba = null; bap2p = bp2p; g = group; whoAmI = w; timeout = to; obs = o; sentByOracle = false; }
        }

        internal static List<byte[]>[] doMultiQuery(List<Group> glist, int nreplies, bool sentByOracle, params object[] obs)
        {
            Timeout timeout;
            splitObs(glist, out timeout, ref obs);
            int ng, gn;
            ng = glist.Count();
            gn = 0;
            List<byte[]>[] ba = new List<byte[]>[ng];
            Thread[] myThreads = new Thread[ng];
            // This is kind of gross but easier than the alternative.  Probably should consider recoding it
            if ((IsisSystem.Debug & (IsisSystem.REPLYWAIT | IsisSystem.VIEWWAIT)) != 0)
            {
                string s = "Doing a MultiQuery in groups ";
                foreach (Group g in glist)
                    s += g.gname + " ";
                Isis.WriteLine(s);
            }
            foreach (Group g in glist)
            {
                myThreads[gn] = new Thread(new ParameterizedThreadStart(myQuerier));
                myThreads[gn].Name = "ISIS MultiQuery thread for <" + g.gname + ">";
                querierArgs qa = new querierArgs(ba, g, nreplies, sentByOracle, gn, timeout, obs);
                myThreads[gn].Start(qa);
                ++gn;
            }

            foreach (Thread t in myThreads)
                t.Join();
            return ba;
        }

        internal static List<byte[]> doMultiP2PQuery(List<Group> glist, params object[] obs)
        {
            Timeout timeout;
            splitObs(glist, out timeout, ref obs);
            int ng, gn;
            ng = glist.Count();
            gn = 0;
            List<byte[]> ba = new List<byte[]>();
            Thread[] myThreads = new Thread[ng];
            // This is kind of gross but easier than the alternative.  Probably should consider recoding it
            if ((IsisSystem.Debug & (IsisSystem.REPLYWAIT | IsisSystem.VIEWWAIT)) != 0)
            {
                string s = "Doing a MultiP2PQuery in groups ";
                foreach (Group g in glist)
                    s += g.gname + " ";
                Isis.WriteLine(s);
            }
            foreach (Group g in glist)
            {
                myThreads[gn] = new Thread(new ParameterizedThreadStart(myQuerier));
                myThreads[gn].Name = "ISIS MultiP2PQuery thread for <" + g.gname + ">";
                querierArgs qa = new querierArgs(ba, g, gn, timeout, obs);
                myThreads[gn].Start(qa);
                ++gn;
            }

            foreach (Thread t in myThreads)
                t.Join();
            return ba;
        }

        private static void myQuerier(Object o)
        {
            querierArgs qa = (querierArgs)o;
            try
            {
                if ((IsisSystem.Debug & (IsisSystem.REPLYWAIT | IsisSystem.VIEWWAIT)) != 0)
                    Isis.WriteLine("before multiQuery(" + qa.g.gname + "): index[" + qa.whoAmI + "] ... nreplies " + qa.nr);
                if (qa.ba != null)
                    qa.ba[qa.whoAmI] = qa.g.doQueryToBA(qa.sentByOracle, qa.nr, qa.timeout, qa.obs);
                else
                    qa.bap2p[qa.whoAmI] = qa.g.doP2PQuery(qa.g.theView.members[0], qa.timeout, qa.obs);
                if ((IsisSystem.Debug & (IsisSystem.REPLYWAIT | IsisSystem.VIEWWAIT)) != 0)
                    Isis.WriteLine("after multiQuery(" + qa.g.gname + "): index[" + qa.whoAmI + "] reply len " + ((qa.ba == null) ? qa.bap2p[qa.whoAmI].Count() : qa.ba[qa.whoAmI].Count()));
            }
            catch (IsisShutdown) { qa.ba[qa.whoAmI] = new List<byte[]>(); }
            IsisSystem.ThreadTerminationMagic();
        }

        internal void ReplayToDo()
        {
            Semaphore ReplayWait = new Semaphore(0, int.MaxValue);
            using (new LockAndElevate(ToDoLock))
            {
                if (ToDo.Count() == 0)
                    return;
                Thread t = new Thread(delegate()
                {
                    try
                    {
                        List<Msg> oldToDo = null;
                        int vid = 0;
                        using (new LockAndElevate(ToDoLock))
                        {
                            oldToDo = ToDo;
                            ToDo = new List<Msg>();
                        }
                        if (oldToDo.Count() > 0)
                        {
                            foreach (Msg m in oldToDo)
                            {
                                using (new LockAndElevate(ViewLock))
                                    if (theView != null)
                                        vid = theView.viewid;
                                if (m.vid > vid || GotAMsg(m, Msg.FIFOCAST, "replayToDo") == false)
                                    using (new LockAndElevate(ToDoLock))
                                    {
                                        if (m.toDoTime == 0)
                                            m.toDoTime = Isis.NOW();
                                        else if ((Isis.NOW() - m.toDoTime) > Isis.ISIS_DEFAULTTIMEOUT * 6)
                                            throw new IsisException("Msg(s) trapped on ToDo list for too long");
                                        ToDo.Add(m);
                                    }
                            }
                        }
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    IsisSystem.ThreadTerminationMagic();
                    ReplayWait.Release(1);
                });
                t.Name = "ToDo Replay Thread";
                t.Start();
            }
            ReplayWait.WaitOne();
        }

        internal void CheckCausalWaitQueue()
        {
            using (new LockAndElevate(CausalOrderListLock))
            {
                ctuple ct = CausalOrderList.FirstOrDefault();
                if (ct != null && (Isis.NOW() - ct.whenEnqueued) > Isis.ISIS_DEFAULTTIMEOUT * 6)
                    throw new IsisException("Isis causal send: message(s) trapped on the causal delivery queue for too long");
            }
        }

        internal int getMinStable()
        {
            if (!HasFirstView)
                return 0;
            return getMinStable(theView);
        }

        internal int getMinStable(View v)
        {
            int ms;
            if ((flags & G_ISLARGE) == 0)
                using (new LockAndElevate(ViewLock))
                    ms = nextMsgid - 1;
            else
                ms = v.NextIncomingMsgID[1] - 1;
            using (new LockAndElevate(ReliableSender.PendingSendBufferLock))
            {
                if ((flags & G_ISLARGE) == 0)
                {
                    foreach (ReliableSender.MsgDesc md in ReliableSender.PendingSendBuffer)
                        if (md.group == this && md.localSender && md.MsgVid == v.viewid)
                        {
                            ms = md.MsgId - 1;
                            break;
                        }
                }
                else
                {
                    foreach (ReliableSender.MsgDesc lgmd in ReliableSender.LgPendingSendBuffer)
                        if (lgmd.group == this)
                        {
                            ms = lgmd.MsgId - 1;
                            break;
                        }
                    if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                        Isis.WriteLine("getMinStable(gname=" + gname + " (large), nextMsgid=" + theView.NextIncomingMsgID[1] + ") returning " + ms);
                }
            }
            return Math.Max(ms, -1);
        }

        internal int StabilityCbPending = -1;

        internal void isStable(Address who, int n)
        {
            if (who == null)
                return;
            using (new LockAndElevate(ViewLock))
            {
                if (theView != null)
                {
                    int rank = theView.isLarge ? -1 : theView.GetRankOf(who);
                    if (rank != -1)
                        theView.StableTo[rank + 1] = Math.Max(theView.StableTo[rank + 1], n);
                }
            }
            using (new LockAndElevate(UnstableLock))
            {
                List<Msg> tmpUnstable = new List<Msg>();
                foreach (Msg m in Unstable)
                    if (m.msgid > n || m.sender.Equals(who) == false)
                        tmpUnstable.Add(m);
                    else if (m.sender.isMyAddress() && m.dest.isGroupAddress() && m.type != Msg.STABILITYINFO)
                        using (new LockAndElevate(GroupFlagsLock))
                            SendStabilityNeeded = true;
                Unstable = tmpUnstable;
            }
        }

        internal static void SendStability()
        {
            List<Group> wantsStabilitySent = new List<Group>();
            List<Group> igc = Group.IsisGroupsClone();
            foreach (Group g in igc)
                using (new LockAndElevate(g.GroupFlagsLock))
                {
                    if ((g.flags & Group.G_SENDINGSTABILITY) == 0 && ((g.SendStabilityNeeded || (g.MaxBacklogSent > Isis.ISIS_ASYNCMTOTALLIMIT / 2)) && (Isis.NOW() - g.SentStableAt) > 250))
                    {
                        g.MaxBacklogSent = 0;
                        g.SendStabilityNeeded = false;
                        wantsStabilitySent.Add(g);
                    }
                }
            foreach (Group g in wantsStabilitySent)
            {
                bool sentIt = ReliableSender.doSendStability(g);
                using (new LockAndElevate(g.GroupFlagsLock))
                    g.SendStabilityNeeded = !sentIt;
            }
        }

        /// <exclude>
        /// <summary>
        /// Internal for use by Isis; public only to satisfy C# scope rules.
        /// </summary>
        /// </exclude>
        [AutoMarshalled]
        public class FlushAggKey
        {
            /// <exclude>
            /// <summary>
            /// Internal for use by Isis
            /// </summary>
            /// </exclude>
            public Address who;

            /// <exclude>
            /// <summary>
            /// Internal for use by Isis
            /// </summary>
            /// </exclude>
            public int state;

            internal FlushAggKey(Address a, int s)
            {
                who = a;
                state = s;
            }

            /// <exclude>
            /// <summary>
            /// Internal for use by Isis AutoMarshaller
            /// </summary>
            /// </exclude>
            public FlushAggKey()
            {
            }

            /// <exclude>
            /// <summary>
            /// Internal for use by Isis
            /// </summary>
            /// <returns>string encoding the state of the flush aggregator</returns>
            /// </exclude>
            public override string ToString()
            {
                return who.ToString() + ((state == tokenInfo.SETSTABLETO) ? "|1" : "|0");
            }

            /// <exclude>
            /// <summary>
            /// Equality comparison
            /// </summary>
            /// <param name="obj">comparison target</param>
            /// </exclude>
            public override bool Equals(object obj)
            {
                if (obj == null || !obj.GetType().Equals(typeof(FlushAggKey)))
                    return false;
                return who.Equals(((FlushAggKey)obj).who) && state == ((FlushAggKey)obj).state;
            }

            /// <exclude>
            /// <summary>
            /// Required hashcode method
            /// </summary>
            /// <returns>the hash code</returns>
            /// </exclude>
            public override int GetHashCode()
            {
                return who.GetHashCode() + (state * 1717);
            }
        }

        /// <exclude>
        /// <summary>
        /// Internal, represents the tokens employed in the Isis token-tree algorithm.  Declared public to comply with C# scoping rules.
        /// </summary>
        /// </exclude>
        public class tokenInfo
        {
            // Wire portion only includes these fields
            internal Address gaddr;              // The group in which this token is circulating
            internal Address groupOwner;         // The sender for that group, used because the members don't track
            internal Address sender;             // Most recent sender, useful for debugging
            internal Isis.ViewDelta[] viewDeltas;// Recent View Deltas, sent as an vector when some kind of membership event occurs
            volatile internal int logicalClock;
            internal int state;                  // State currently has just 3 values
            internal int stableAtSender;               // Stable up to this msgid. 
            internal int viewid;                 // Viewid for this token
            internal int alsoSeenBase;           // alsoSeen is computed wrt to this base
            internal long alsoSeen;              // Up to 64 bits for ids > stable that have been seen membership in large groups
            internal int tokenLevel;             // Level of the token tree at which this token was sent
            internal int aggStable;              // Aggregated stable value for nodes within the specified level, for transmission
            internal int stableTo;               // Propagates out from the group leader and triggers garbage collection
            internal byte[][] incomingValuesArray; // Extracted on incoming token, undefined for outgoing token

            // State values
            internal const int NORMAL = 0;      // Large group in the normal operational mode
            internal const int INQUIRY = 1;     // New leader inquiring about member states
            internal const int SETSTABLETO = 2; // New leader is promulgating the initial "stableto" value for its first view

            // Not transmitted (inferred on token arrival)
            internal const int RINGSIZE = 8;    // 25 worked well in Quicksilver Scalable Multicast.  Must be >= 2!  Guess: optimal is log(N)

            // Per-group values used within the token algorithms    
            internal long whenReceived;          // Time when this token was received
            volatile internal bool inhibitResenderLoop;   // While stabilizing after a large-group membership change, inhibits resender loop temporarily
            internal long gotAllAt;              // ... associated delay timer
            internal int unstableVIDMID;         // Multicast ID number that was used to cast a view id that isn't stable yet
            internal int unstableVID;            // Associated viewID
            internal int stableVID;              // Max view id to have become stable so far
            internal LockObject slock = new LockObject("token.slock");// Protects the VID fields
            internal Group theGroup;
            internal View WorkingView;
            internal long WorkingViewInstalledAt;
            internal bool IAmLgOwner = false;
            internal bool[] IAmRank0;
            internal ILock FlushingBarrier;      // Used to wait while flushing is running
            internal LockObject FlushingBarrierLock = new LockObject("token.FlushingBarrierLock");  // Better safe than sorry!
            internal int nlevels;
            internal int[] mySubgroupIdx;
            internal int[] myOffset;
            internal int[] StableByLevel;           // Used in the rank-0 members of each ring, StableByLevel[i] is the value of aggStable received from the last guy in ring [i]
            internal int[] includeViewDeltas;       // Tells me if the view delta vector for this level needs to be included
            internal tokenInfo[] lastToken;         // Last token I received 
            internal Address[] next;
            internal Address[] last;
            internal bool[] lastValidated;          // I've received at least one token from last since prior reset
            internal bool[] sentAToken;             // True if I sent a token to try and "push" last[i] into sync with me
            internal bool[] pinged;                 // True if I didn't get a token from you so I pinged you
            internal int[] tokenInMotion;           // Counts the number of tokens to next[level] that are "in motion"
            internal LockObject tokenInMotionLock = new LockObject("tokenInMotionLock");
            internal long resetTime;

            internal tokenInfo(Group g)
            {
                if (g.theToken == null || g.theToken.viewDeltas == null)
                {
                    viewDeltas = new Isis.ViewDelta[0];
                    WorkingView = g.theView;
                }
                else
                {
                    viewDeltas = g.theToken.viewDeltas;
                    WorkingView = g.theToken.WorkingView;
                }
                WorkingViewInstalledAt = Isis.NOW();
                theGroup = g;
                reinitializeToken(g);
                IAmLgOwner = (g.theView.GetMyRank() == 0);
            }

            // Caller must hold g.Lock and g.tokenLock
            internal static void newToken(Group g)
            {
                tokenInfo oldToken;
                oldToken = g.theToken;
                g.theToken = new tokenInfo(g);
                if (oldToken != null)
                {
                    g.theToken.stableTo = oldToken.stableTo;
                    g.theToken.logicalClock = oldToken.logicalClock + 1;
                }
                if (oldToken == null || oldToken.theGroup.AggList == null)
                    return;
                g.theToken.logicalClock = oldToken.logicalClock;
                if (oldToken.IAmLgOwner)
                    for (int n = 0; n < oldToken.theGroup.AggList.Length; n++)
                        foreach (IAggregateEventHandler ae in oldToken.theGroup.AggList[n])
                            ae.AggEvent(Group.BreakWaits);
                else if ((IsisSystem.Debug & IsisSystem.TOKENFLUSH) != 0)
                    Isis.WriteLine("<" + g.gname + ">: Unhibit breakwaits (I wasn't the previous LgOwner");
            }

            internal void resetStableByLevel(Group g)
            {
                if ((IsisSystem.Debug & IsisSystem.TOKENSTABILITY) != 0)
                    Isis.WriteLine("<" + g.gname + ">: LogicalClock=" + logicalClock + ", reset StableByLevel[*]=-1");
                for (int level = 0; level < StableByLevel.Length; level++)
                    StableByLevel[level] = -1;
            }

            // Caller holds g.tokenLock
            internal void reinitializeToken(Group g)
            {
                Address[] Mlist = WorkingView.members;
                int NMemb = Mlist.Length, myrank = WorkingView.GetMyRank();
                if (NMemb == 0 || myrank == -1)
                    return;

                nlevels = 0;
                for (int n = NMemb; n > 0; n /= RINGSIZE)
                    nlevels++;

                if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                    Isis.WriteLine(" REINITIALIZE TOKEN FOR GROUP <" + theGroup.gname + ">, view " + WorkingView.viewid + " NMemb=" + NMemb + " nlevels=" + nlevels);

                resetTime = Isis.NOW();
                mySubgroupIdx = new int[nlevels];
                myOffset = new int[nlevels];
                next = new Address[nlevels];
                last = new Address[nlevels];
                lastValidated = new bool[nlevels];
                sentAToken = new bool[nlevels];
                pinged = new bool[nlevels];
                ReliableSender.CleanLgCallbacks(g);
                using (new LockAndElevate(tokenInMotionLock))
                    tokenInMotion = new int[nlevels];
                StableByLevel = new int[nlevels];
                lastToken = new tokenInfo[nlevels];
                includeViewDeltas = new int[nlevels];
                IAmRank0 = new bool[nlevels];
                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("Reinitialize the AggList[nlevels=" + nlevels + "] from the AggTypes List in <" + g.gname + ">... (it lists " + g.AggTypes.Count() + " types)");
                theGroup.AggList = new LinkedList<object>[nlevels];
                viewid = WorkingView.viewid;
                using (new LockAndElevate(theGroup.AggListLock))
                {
                    binfo.resetBarrierList();
                    for (int n = 0; n < nlevels; n++)
                    {
                        theGroup.AggList[n] = new LinkedList<object>();
                        foreach (AggInfo ag in g.AggTypes)
                        {
                            if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                                Isis.WriteLine("Calling constructor in <" + g.gname + "> to allocate a new aggregator of type " + ag.KVT);
                            // These are actually "aggregator" objects of some derived type
                            theGroup.AggList[n].AddLast((object)(ag.myFactory.Invoke(new Object[] { ag.theGroup, viewid, n, ag.theDel, ag.theTimeout })));
                        }
                    }
                }
                if (viewDeltas.Length != 0)
                    for (int n = 0; n < nlevels; n++)
                        includeViewDeltas[n] = 4;

                groupOwner = Mlist[0];
                stableAtSender = aggStable = stableTo = -1;
                gaddr = theGroup.gaddr;
                sender = Isis.my_address;

                bool inNextLevel = true;
                int stride = 1, ringsize = RINGSIZE;
                for (int i = 0; i < nlevels; i++)
                {
                    if (inNextLevel)
                    {
                        int residue = -1;
                        for (int delta = 0; delta < 15; delta++)
                        {
                            int theResidue = NMemb % (ringsize - delta);
                            if (theResidue == 0)
                                break;
                            else if (theResidue > residue)
                                residue = NMemb % (ringsize - delta);
                        }
                        mySubgroupIdx[i] = myrank / ringsize;
                        myOffset[i] = myrank % ringsize;
                        IAmRank0[i] = (myOffset[i] == 0);
                        int sgBase = myrank - myOffset[i];
                        int idx = sgBase + (myOffset[i] + stride) % ringsize;
                        if (idx >= NMemb)
                            idx = sgBase;
                        next[i] = Mlist[idx];
                        int lidx = myrank - stride;
                        if (lidx < sgBase)
                        {
                            lidx += ringsize;
                            while (lidx >= NMemb)
                                lidx -= stride;
                        }
                        last[i] = Mlist[lidx];
                        lastValidated[i] = false;
                        sentAToken[i] = false;
                        if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                            Isis.WriteLine("Token layer[" + i + "] initializing next/last: MemberList = " + Address.VectorToString(Mlist) + ", myRank=" + myrank + ", ringsize=" + ringsize + ", sgBase=" + sgBase + ", last[" + i + "]=" + last[i] + ", next[" + i + "]=" + next[i]);
                        inNextLevel = (myrank % ringsize == 0);
                        stride = ringsize;
                        ringsize *= RINGSIZE;
                    }
                    else
                    {
                        if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                            Isis.WriteLine("Token layer[" + i + "] not in at this level, setting nlevels=" + i);
                        nlevels = i;
                    }
                    alsoSeen = 0;
                    StableByLevel[i] = -1;
                }

                if (groupOwner.isMyAddress() && (next[nlevels - 1] == null || next[nlevels - 1].isMyAddress()))
                    nlevels = Math.Max(1, nlevels - 1);

                if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                    Isis.WriteLine("... reinitalized token = " + this);
                if ((IsisSystem.Debug & IsisSystem.TOKENSTABILITY) != 0)
                {
                    Isis.WriteLine("<" + g.gname + ">: New view = " + Address.VectorToString(Mlist) + "\r\n");
                    for (int level = 0; level < next.Length; level++)
                    {
                        Isis.WriteLine("<" + g.gname + ">:   -- last[" + level + "]=" + (last[level] == null ? "(null)" : (last[level].isMyAddress() ? "(self)" : last[level].ToString())));
                        Isis.WriteLine("<" + g.gname + ">:   -- next[" + level + "]=" + (next[level] == null ? "(null)" : (next[level].isMyAddress() ? "(self)" : next[level].ToString())));
                    }
                }
            }

            // Used to generate a working view by applying view deltas to the group view
            // Parent group is the group within which the action is occuring, but this call can
            // occur far into the future (it mostly updates the WorkingView structure) and we
            // don't do callbacks at this time.  So we just create a temporary, fake, Group object
            // Returns the first group view in which I am the leader (or -1, if none); used in flush
            internal void applyViewDeltas(Group parentGroup, Isis.ViewDelta[] newvds)
            {
                using (new LockAndElevate(parentGroup.TokenLock))
                using (new LockAndElevate(parentGroup.ViewLock))
                {
                    int priorView = -1;
                    if (WorkingView != null)
                        priorView = WorkingView.viewid;
                    updateViewDeltas(newvds);
                    using (Group g = new Group())
                    {
                        g.AggTypes = parentGroup.AggTypes;
                        g.gaddr = parentGroup.gaddr;
                        g.gname = parentGroup.gname;
                        using (new LockAndElevate(g.ViewLock))
                            g.theView = WorkingView;
                        foreach (Isis.ViewDelta vd in viewDeltas)
                        {
                            if (vd.prevVid == g.theView.viewid)
                            {
                                if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                                    Isis.WriteLine("  applying view delta = " + vd);
                                Isis.UpdateGroupView(false, vd, g, "update theToken.WorkingView");
                                if (parentGroup.myFirstLeadershipView == 0 && g.IAmRank0())
                                    parentGroup.myFirstLeadershipView = g.theView.viewid;
                            }
                        }
                        WorkingView = g.theView;
                        WorkingViewInstalledAt = Isis.NOW();
                        if (parentGroup.gaddr != null && parentGroup.gaddr.Equals(Isis.ISISMEMBERS.gaddr))
                            IPMCNewView(parentGroup.gaddr, WorkingView);
                    }
                    if (priorView == WorkingView.viewid)
                        return;
                    // Finally, recompute the token using the new WorkingView 
                    if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                        Isis.WriteLine("reinitializing the token from the working view = " + WorkingView);
                    newToken(parentGroup);
                }
            }

            internal void updateViewDeltas(Isis.ViewDelta[] newvds)
            {
                if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                {
                    Isis.WriteLine("TOKEN: updateViewDeltas called on a VD vector of length " + newvds.Length + ", with prior VDS = ");
                    foreach (Isis.ViewDelta vd in viewDeltas)
                        Isis.WriteLine("   " + vd);
                    Isis.WriteLine("NEW VIEW DELTAS TO MERGE IN:");
                    foreach (Isis.ViewDelta vd in newvds)
                        Isis.WriteLine("   " + vd);
                }

                // First create one merged list of view deltas
                List<Isis.ViewDelta> newvdlist = new List<Isis.ViewDelta>();
                foreach (Isis.ViewDelta vd in newvds)
                {
                    bool fnd = false;
                    if (vd.prevVid < stableVID)
                        fnd = true;
                    else foreach (Isis.ViewDelta knownvd in viewDeltas)
                            if (vd.prevVid <= knownvd.prevVid || (knownvd.leaderId == vd.leaderId && knownvd.gaddr.Equals(vd.gaddr) && knownvd.prevVid == vd.prevVid))
                            {
                                fnd = true;
                                break;
                            }

                    if (!fnd)
                        foreach (Isis.ViewDelta knownvd in newvdlist)
                            if (knownvd.leaderId == vd.leaderId && knownvd.gaddr.Equals(vd.gaddr) && knownvd.prevVid == vd.prevVid)
                            {
                                fnd = true;
                                break;
                            }
                    if (!fnd)
                        newvdlist.Add(vd);
                }
                if (newvdlist.Count() == 0)
                    return;

                using (new LockAndElevate(slock))
                {
                    Isis.ViewDelta[] mergedVds = new Isis.ViewDelta[viewDeltas.Length + newvdlist.Count()];
                    int idx = 0, maxvid = -1;
                    foreach (Isis.ViewDelta vd in viewDeltas)
                    {
                        mergedVds[idx++] = vd;
                        maxvid = vd.prevVid;
                    }

                    foreach (Isis.ViewDelta vd in newvdlist)
                        if (vd.prevVid >= stableVID && vd.prevVid > maxvid)
                            mergedVds[idx++] = vd;

                    viewDeltas = mergedVds;
                    for (int level = 0; level < nlevels; level++)
                        // Deltas will be included on the next few tokens sent
                        includeViewDeltas[level] = 4;

                    fixVDS();

                    // Now apply the vds to create the working view using a fake group 
                    if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                        Isis.WriteLine("TOKEN: updateViewDeltas recomputed VD vector..." + this);
                }
            }

            internal void fixVDS()
            {
                int idx = 0;
                for (int i = 0; i < viewDeltas.Length; i++)
                    if (viewDeltas[i] != null && viewDeltas[i].prevVid >= stableVID)
                        viewDeltas[idx++] = viewDeltas[i];
                if (idx != viewDeltas.Length)
                    Isis.ArrayResize<Isis.ViewDelta>(ref viewDeltas, idx);
            }

            // Checks for holes in the ViewDelta list of the current token; used as a unit-check
            internal void checkVDS(string fromWhere)
            {
                Group g = doLookup(gaddr);
                if (g == null)
                {
                    Isis.WriteLine("Warning: CheckVDS(" + fromWhere + ") -- group lookup returned null");
                    return;
                }
                if ((g.flags & G_ISLARGE) == 0)
                    return;
                tokenInfo theToken;
                using (new LockAndElevate(g.TokenLock))
                    theToken = g.theToken;
                View theView;
                using (new LockAndElevate(g.ViewLock))
                    theView = g.theView;
                if (theToken == null || theView == null)
                {
                    Isis.WriteLine("Warning: CheckVDS(" + fromWhere + ") -- theToken null or theView null");
                    return;
                }
                if (theToken != this)
                {
                    Isis.WriteLine("Warning: CheckVDS(" + fromWhere + ") -- theToken != g.theToken");
                    return;
                }
                if (theToken.viewDeltas == null || theToken.viewDeltas.Length == 0)
                    return;
                Isis.ViewDelta vd = theToken.viewDeltas[0];
                if (vd.prevVid > theView.viewid)
                    throw new IsisException("in checkVDS(" + fromWhere + ") vd[0].previd=" + vd.prevVid + " but theView=" + theView);
                View wv = theToken.WorkingView;
                if (vd.prevVid > wv.viewid)
                    throw new IsisException("in checkVDS(" + fromWhere + ") vd[0].previd=" + vd.prevVid + " but WorkingView=" + wv);
            }

            /// <summary>
            /// Constructor for tokens received via Isis Msg layer
            /// </summary>
            /// <param name="ba">byte vector encoding a token</param>
            public tokenInfo(byte[] ba)
            {
                int idx = 0;
                object[] obs = Msg.BArrayToObjects(ba, typeof(Address), typeof(Address), typeof(Address),
                    typeof(int), typeof(int), typeof(int), typeof(int), typeof(int), typeof(long), typeof(int),
                    typeof(int), typeof(int), typeof(int), typeof(Isis.ViewDelta[]), typeof(byte[][]));
                gaddr = (Address)obs[idx++];
                theGroup = Group.doLookup(gaddr);
                groupOwner = (Address)obs[idx++];
                sender = (Address)obs[idx++];
                logicalClock = (int)obs[idx++];
                state = (int)obs[idx++];
                viewid = (int)obs[idx++];
                stableAtSender = (int)obs[idx++];
                alsoSeenBase = (int)obs[idx++];
                alsoSeen = (long)obs[idx++];
                aggStable = (int)obs[idx++];
                tokenLevel = (int)obs[idx++];
                stableTo = (int)obs[idx++];
                stableVID = (int)obs[idx++];
                viewDeltas = (Isis.ViewDelta[])obs[idx++];
                incomingValuesArray = (byte[][])obs[idx++];
                if (theGroup != null && theGroup.myAes != null)
                {
                    using (new LockAndElevate(theGroup.myAesLock))
                    {
                        if ((IsisSystem.Debug & IsisSystem.CIPHER) != 0)
                            foreach (byte[] bbv in incomingValuesArray)
                                dumpBv("\r\n*************\r\nRecived, decipher from " + sender, bbv);
                        for (int i = 0; i < incomingValuesArray.Length; i++)
                            incomingValuesArray[i] = theGroup.decipherBuf(incomingValuesArray[i]);
                        if ((IsisSystem.Debug & IsisSystem.CIPHER) != 0)
                            foreach (byte[] bbv in incomingValuesArray)
                                dumpBv("After decipher from " + sender, bbv);
                    }
                }
            }

            internal static void dumpBv(string where, byte[] bv)
            {
                string s = " { ";
                foreach (byte b in bv)
                    s += b.ToString("X2") + " ";
                s += "} ";
                Isis.WriteLine(where + s);
            }

            /// <summary>
            /// Marshall token to out form
            /// </summary>
            /// <returns>byte array encoding the token</returns>
            public byte[] toBArray()
            {
                byte[][] aggBv = aggsToBArray(tokenLevel, IAmRank0[tokenLevel]);

                if (theGroup != null && theGroup.myAes != null)
                {
                    using (new LockAndElevate(theGroup.myAesLock))
                    {
                        if ((IsisSystem.Debug & IsisSystem.CIPHER) != 0)
                            foreach (byte[] bbv in aggBv)
                                dumpBv("\r\n============\r\nBefore encipher toBArray in " + Isis.my_address, bbv);
                        for (int i = 0; i < aggBv.Length; i++)
                            aggBv[i] = theGroup.cipherBuf(aggBv[i]);
                        if ((IsisSystem.Debug & IsisSystem.CIPHER) != 0)
                            foreach (byte[] bbv in aggBv)
                                dumpBv("After encipher   in " + Isis.my_address, bbv);
                    }
                }
                Isis.ViewDelta[] vds;
                if (includeViewDeltas[tokenLevel]-- > 0)
                    vds = viewDeltas;
                else
                    vds = new Isis.ViewDelta[0];
                byte[] bv = Msg.toBArray(gaddr, groupOwner, sender, logicalClock, state, viewid, stableAtSender, alsoSeenBase, alsoSeen, aggStable, tokenLevel, stableTo, stableVID, vds, aggBv);
                return bv;
            }

            internal byte[][] aggsToBArray(int level, bool fromRank0Member)
            {
                List<byte[]> bas = new List<byte[]>();
                using (new LockAndElevate(theGroup.AggListLock))
                    if (theGroup.AggList[level] != null)
                        foreach (IAggregateEventHandler ae in theGroup.AggList[level])
                            bas.Add(ae.AggEvent(fromRank0Member ? GetRank0State : GetAggState));
                return bas.ToArray();
            }

            /// <exclude>
            /// <summary>
            /// Internal: Pretty-print an Isis token.  Must be public to satisfy C# scoping rules.
            /// </summary>
            /// <returns>string for the token</returns>
            /// </exclude>
            public override string ToString()
            {
                Group g = Group.doLookup(gaddr);
                string flushing = (state == INQUIRY) ? "FLUSH:INQUIRY " : ((state == SETSTABLETO) ? "FLUSH:SETSTABLETO " : "");
                string s = flushing + "TOKEN[level=" + tokenLevel + ":" + logicalClock + " from " + sender + "]: ViewID " + viewid + " of group <" + ((g == null) ? gaddr.ToString() : g.gname) + ">, groupOwner " + groupOwner;
                if (viewDeltas.Length == 0)
                    s += "(No View Delta vector)";
                if (WorkingView != null)
                    s += "\r\n         WorkingView = " + WorkingView.ViewSummary();
                string sbl = " ";
                for (int level = 0; level < nlevels; level++)
                    sbl += "[" + level + "]=" + StableByLevel[level] + " ";
                s += "\r\n         StableTo=" + stableTo + ", aggStable=" + aggStable + ", StableVID=" + stableVID + ", stableAtSender=" + stableAtSender + ", alsoSeenBase = " + alsoSeenBase + ", alsoSeen=" + alsoSeen.ToString("X") + ", StableByLevel = {" + sbl + "}";
                s += AggState(this);
                if (viewDeltas.Length > 0)
                {
                    if (includeViewDeltas != null)
                    {
                        s += "\r\n         ...  includeViewDeltas[";
                        for (int b = nlevels; b > 0; b--)
                            s += " " + (includeViewDeltas[b - 1] > 0);
                        s += " ]";
                    }
                    s += "\r\n         ...  vector of view deltas:";
                    foreach (Isis.ViewDelta vd in viewDeltas)
                        s += "\r\n         " + vd;
                }
                for (int i = 0; i < nlevels; i++)
                    s += "\r\n         level[" + i + "] info: mySubgroupIdx=" + mySubgroupIdx[i] + ", myOffset=" + myOffset[i] + ", StableByLevel=" + StableByLevel[i] +
                        ("\r\n              gossips to " + (next[i] == null ? "(null)" : ((next[i].isMyAddress() ? "self" : next[i].ToString())))) +
                        (", receives gossip from " + (last[i] == null ? "(null)" : ((last[i].isMyAddress() ? "self" : last[i].ToString())))) +
                        (", last received token was from " + (lastToken[i] == null ? "(none)" : (lastToken[i].sender + " and arrived at time " + Isis.TimeToString(lastToken[i].whenReceived)))) +
                        (tokenInMotion[i] > 0 ? (" (" + tokenInMotion[i] + " outgoing tokens are in motion)") : "");
                return s;
            }
        }

        internal static string AggState(tokenInfo ti)
        {
            string s = "";
            for (int l = 0; l < ti.nlevels; l++)
                foreach (IAggregateEventHandler ae in ti.theGroup.AggList[l])
                {
                    string astate = ae.AggState();
                    if (astate.Length > 0)
                        s += "            Aggregation state for " + ae.AggName() + " level " + l + "\r\n" + astate;
                }
            if (s.Length > 0)
                return "\r\n         AGGREGATION DATA FOR THIS TOKEN:\r\n" + s;
            return s;
        }

        internal void becomeGroupOwner()
        {
            using (new LockAndElevate(TokenLock))
                if (theToken.IAmLgOwner)
                    return;

            if (myFirstLeadershipView == 0)
                throw new IsisException("becomeGroupOwner yet myFirstLeadershipView was unknown!");
            if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                Isis.WriteLine("Becoming the LargeGroupOwner... for <" + gname + "> as of viewid " + myFirstLeadershipView);

            LgFlush(true);

            if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                Isis.WriteLine("I am now the LargGroupOwner for <" + gname + ">!");
        }

        internal void CheckForFutureViews(Msg m)
        {
            // This peek-ahead code existed way back in Isis too.  Is it actually a necessary feature of
            // an outboard virtual synchrony implementation?  Check to see if this is a large-group relayed NEWVIEW
            // message.  In that specific case, note the viewid on the message.  It may not be delivered promptly.
            // because other prior view messages may be in the queue, but if so those will do LgFlush calls that can't
            // complete (can't stabilize) because they are trying to flush a view that is already stale....
            Object[] obs = Msg.MsgToObjects(m);
            if (obs.Length == 2 && obs[0].GetType().Equals(typeof(int)) && ((int)obs[0]) == Isis.RELAYSEND)
            {
                if ((IsisSystem.Debug & IsisSystem.FLUSHING) != 0)
                {
                    if (obs[1].GetType().Equals(typeof(Isis.ViewDelta[])))
                        Isis.WriteLine("CheckForFutureViews: " + Isis.rToString((int)obs[0]) + ", msg " + m.vid + ":" + m.msgid + " from " + m.sender +
                            "... Found a vd vector, len " + ((Isis.ViewDelta[])obs[1]).Length + "... Starts with previd=" + ((Isis.ViewDelta[])obs[1])[0].prevVid);
                }
                if (obs[1].GetType().Equals(typeof(Isis.ViewDelta[])))
                    gotNewViewDeltas((Isis.ViewDelta[])obs[1]);
            }

        }

        // Core of virtual synchrony algorithm for large groups is embodied into this flush protocol
        // The logic is (WAY) more tricky than it probably looks, because different processes in the group
        // can be at different stages in learning about the most current view.  We can only stabilize the
        // the group if they all have the same view for long enough to circulate a token.
        //
        // This algorithm is actually a variant
        // on the <>W result of Chandra and Toueg, believe it or not... one of those theory results you never
        // thought you would encounter in practice, yet here it is.  The complexity here arises because if a
        // newer view becomes known to the system, the PeekAhead logic (above, in CheckForFutureViews)
        // aborts the current flush and rushes us to try and flush in the most current view possible
        // Thus we can easily end up with multiple pending views and multicasts before we finally manage to
        // hold still long enough to flush on the view with the largest viewid in the sequence.\
        //
        // In practice this could be done only for views that report departures or failures and perhaps doing
        // so would be wise, but right now it kicks in on any view, no matter what it reports
        // Needless to say, thrashing can delay the flush and effectively leave a large group hung until
        // it crashes in some dreadful way.  So thrashing would be a DDOS attack on Isis2.
        //
        internal void LgFlush(bool becomeOwner)
        {
            tokenInfo theToken;
            using (new LockAndElevate(TokenLock))
                theToken = this.theToken;
            using (new ILock(ILock.LLLARGE, gaddr))
            {
                if (theToken.WorkingView.nLive() > 1)
                {
                    if (theView.GetMyRank() == -1 || (!becomeOwner && !theToken.IAmLgOwner))
                        throw new IsisException("LgFlush but I'm either not in the large group, or I lack ownership rights!");

                    if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                        Isis.WriteLine("BEFORE LgFlush **************************************************" + IsisSystem.GetState());
                    using (new LockAndElevate(theToken.FlushingBarrierLock))
                        theToken.FlushingBarrier = ILock.IlockRef(ILock.LLLARGEBW, gaddr);
                    using (new LockAndElevate(TokenLock))
                    {
                        // Set up for initial flush
                        theToken.logicalClock++;
                        theToken.IAmLgOwner = true;
                        theToken.groupOwner = Isis.my_address;
                        theToken.stableTo = -1;
                    }
                    if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                        Isis.WriteLine("In LgFlush before barrier wait... setting theToken.stableTo = -1");

                    // Wait for it to finish
                    if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                        Isis.WriteLine("LgFlush: Before Flush[INQUIRY]!");
                    bool doAgain = false;
                    int limit = theToken.WorkingView.members.Length;
                    do
                    {
                        try
                        {
                            doAgain = false;
                            theToken.state = tokenInfo.INQUIRY;
                            if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                                Isis.WriteLine("Setting Aggregator Value(INQUIRY in vid=" + myFirstLeadershipView + "): " + theToken.groupOwner + ":" + tokenInfo.INQUIRY + " to true ");
                            SetAggregatorValueAndVID<FlushAggKey, bool>(myFirstLeadershipView, new FlushAggKey(theToken.groupOwner, tokenInfo.INQUIRY), true);
                            if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                                Isis.WriteLine("Calling GetAggResult for FLUSHAgg(INQUIRY in vid=" + myFirstLeadershipView + ") myAddress " + Isis.my_address);
                            GetAggregatorResult<FlushAggKey, bool>(new FlushAggKey(Isis.my_address, tokenInfo.INQUIRY));
                        }
                        catch (AggregationFailed)
                        {
                            if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                                Isis.WriteLine("LgFlush: Restarting Flush[INQUIRY]!");
                            if (--limit <= 0)
                                throw new IsisException("Infinite loop in LgFlush");
                            doAgain = true;
                        }
                    }
                    while (doAgain);
                    if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                        Isis.WriteLine("LgFlush: After Flush[INQUIRY]!");

                    using (new LockAndElevate(TokenLock))
                        theToken.stableTo = theToken.StableByLevel[theToken.nlevels - 1];
                    ReliableSender.gotLgAck(this, theToken.stableTo);
                    theToken.resetStableByLevel(this);

                    if (nextMsgid > theView.NextIncomingMsgID[1])
                        throw new IsisException("In LgFlush[1] had nextMsgid=" + nextMsgid + ", but after protocol setting it DOWN to " + theView.NextIncomingMsgID[1] + " in state " + IsisSystem.GetState());
                    using (new LockAndElevate(ViewLock))
                        nextMsgid = theView.NextIncomingMsgID[1];

                    if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                        Isis.WriteLine("In LgFlush after barrier wait... setting theToken.stableTo = " + theToken.stableTo);

                    // Wait for them to all learn the value of stableTo
                    using (new LockAndElevate(TokenLock))
                        theToken.logicalClock++;
                    if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                        Isis.WriteLine("LgFlush: Before Flush[SETSTABLETO]!");
                    do
                    {
                        try
                        {
                            doAgain = false;
                            using (new LockAndElevate(TokenLock))
                                theToken.state = tokenInfo.SETSTABLETO;
                            if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                                Isis.WriteLine("Setting Aggregator Value(STABLETO in vid=" + myFirstLeadershipView + "): " + theToken.groupOwner + ":" + tokenInfo.SETSTABLETO + " to true ");
                            SetAggregatorValueAndVID<FlushAggKey, bool>(myFirstLeadershipView, new FlushAggKey(theToken.groupOwner, tokenInfo.SETSTABLETO), true);
                            if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                                Isis.WriteLine("Calling GetAggResult for FLUSHAgg(SETSTABLETO in vid=" + myFirstLeadershipView + ") myAddress " + Isis.my_address);
                            GetAggregatorResult<FlushAggKey, bool>(new FlushAggKey(Isis.my_address, tokenInfo.SETSTABLETO));
                        }
                        catch (AggregationFailed)
                        {
                            if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                                Isis.WriteLine("LgFlush: Restarting Flush[SETSTABLETO]!");
                            doAgain = true;
                        }
                    }
                    while (doAgain);
                    if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                        Isis.WriteLine("LgFlush: After Flush[SETSTABLETO]!");

                    using (new LockAndElevate(theToken.FlushingBarrierLock))
                    {
                        theToken.FlushingBarrier.BarrierReleaseAll();
                        theToken.FlushingBarrier = null;
                    }
                    if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                        Isis.WriteLine("AFTER LgFlush **************************************************" + IsisSystem.GetState());

                    using (new LockAndElevate(TokenLock))
                    {
                        theToken.logicalClock++;
                        theToken.state = tokenInfo.NORMAL;
                    }
                }
                else
                {
                    if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                        Isis.WriteLine("SKIPPING LgFlush protocol (group only has one member)");
                    theToken.IAmLgOwner = true;
                    theToken.groupOwner = Isis.my_address;
                    if (nextMsgid > theView.NextIncomingMsgID[1])
                        throw new IsisException("In LgFlush[2] had nextMsgid=" + nextMsgid + ", but skipping protocol and setting it DOWN to " + theView.NextIncomingMsgID[1] + " in state " + IsisSystem.GetState());

                    using (new LockAndElevate(ViewLock))
                    {
                        nextMsgid = theView.NextIncomingMsgID[1];
                        theToken.stableTo = nextMsgid - 1;
                    }
                    ReliableSender.gotLgAck(this, theToken.stableTo);
                }
            }
            if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                Isis.WriteLine("After LgFlush nextMsgId=" + theToken.viewid + ":" + nextMsgid);
        }

        // Called each time the large group has a view change.  This causes the group to reapply any future view deltas it knows about
        internal void resetAggregates()
        {
            tokenInfo theToken;
            using (new LockAndElevate(TokenLock))
                theToken = this.theToken;
            if (theToken.WorkingView != null && theToken.WorkingView.viewid > theView.viewid)
                throw new IsisException("resetAggregates");
            theToken.WorkingView = theView;
            theToken.WorkingViewInstalledAt = Isis.NOW();
            theToken.applyViewDeltas(this, new Isis.ViewDelta[0]);
            updateSeenInfo();
        }

        internal void updateSeenInfo()
        {
            tokenInfo theToken;
            View theView;
            using (new LockAndElevate(TokenLock))
            using (new LockAndElevate(ViewLock))
            {
                theToken = this.theToken;
                theView = this.theView;
            }
            long alsoSeen = 0;
            int myStable, alsoSeenBase;
            if (theToken.IAmLgOwner)
                alsoSeenBase = myStable = theView.NextIncomingMsgID[1] - 1;
            else
            {
                using (new LockAndElevate(ReliableSender.PendingSendBufferLock))
                {
                    myStable = theView.NextIncomingMsgID[1] - 1;
                    alsoSeenBase = theToken.stableTo;
                    foreach (ReliableSender.MsgDesc lgmd in ReliableSender.LgPendingSendBuffer)
                        if (lgmd.group == this && lgmd.MsgId > alsoSeenBase && (lgmd.MsgId - alsoSeenBase - 1) < 64)
                            alsoSeen |= (1L) << (lgmd.MsgId - alsoSeenBase - 1);
                }
            }
            if (theToken.stableAtSender != myStable || theToken.alsoSeenBase != alsoSeenBase || theToken.alsoSeen != alsoSeen)
            {
                theToken.stableAtSender = myStable;
                theToken.alsoSeenBase = alsoSeenBase;
                theToken.alsoSeen = alsoSeen;
                if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                {
                    string s = "";
                    foreach (Msg m in Unstable)
                        s += " " + m.vid + ":" + m.msgid;
                    Isis.WriteLine("updateSeenInfo: maxStable: " + myStable + ", unstable msgs: {" + s + " }, setting alsoSeenBase = " + alsoSeenBase + ", alsoSeen=" + alsoSeen.ToString("X"));
                }
            }
        }

        internal bool watchingForStableTo = false;

        // In this method, be careful not to touch fields of the incoming token, "toke", that aren't in the
        // transmitted field group.  Those won't be initialized and hence have default values.
        internal void processIncoming(Address who, tokenInfo toke)
        {
            List<ReliableSender.MsgDesc> toForwardFromPSB = new List<ReliableSender.MsgDesc>();
            using (new LockAndElevate(ReliableSender.PendingSendBufferLock))
                foreach (ReliableSender.MsgDesc lgmd in ReliableSender.LgPendingSendBuffer)
                    if (lgmd.group == this)
                        toForwardFromPSB.Add(lgmd);
            ForwardMsg(toke, toForwardFromPSB);
            interesting = true;
            try
            {
                if (toke.viewDeltas != null && toke.viewDeltas.Length != 0)
                    gotNewViewDeltas(toke.viewDeltas);
                using (new LockAndElevate(TokenLock))
                {
                    bool valid = true;
                    string because = "";
                    View theView;
                    using (new LockAndElevate(ViewLock))
                        theView = this.theView;
                    if (theView.viewid > toke.viewid)
                    {
                        if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                        {
                            because = "processIncoming: INVALID Token was for view " + toke.viewid + " of group " + gname + " but I'm on view " + theView.viewid;
                            Isis.WriteLine(because);
                        }
                        valid = false;
                    }
                    bool switchOwner = false;
                    if (toke.groupOwner.Equals(theToken.groupOwner) == false)
                    {
                        if (theView.GetRawRankOf(theToken.groupOwner) >= theView.GetRawRankOf(toke.groupOwner))
                        {
                            if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                            {
                                because = "processIncoming: INVALID Token for " + toke.gaddr + " shows owner " + toke.groupOwner + ", but I think " + theToken.groupOwner + " should be the owner!";
                                Isis.WriteLine(because);
                            }
                            valid = false;
                        }
                        else
                            switchOwner = true;
                    }

                    if (toke.logicalClock < theToken.logicalClock && theView.GetRawRankOf(theToken.groupOwner) >= theView.GetRawRankOf(toke.groupOwner))
                    {
                        if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                        {
                            because = "processIncoming: INVALID Token for " + toke.gaddr + " had logical clock " + toke.logicalClock + ", but I think the logical clock should read " + theToken.logicalClock + "!";
                            Isis.WriteLine(because);
                        }
                        valid = false;
                    }

                    if (toke.tokenLevel > theToken.last.Length || theToken.last[toke.tokenLevel] == null || !toke.sender.Equals(theToken.last[toke.tokenLevel]))
                    {
                        if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                        {
                            because = "processIncoming: INVALID Token for " + toke.gaddr + " had level " + toke.tokenLevel + ", but I didn't expect to receive a token from this source at that level!  For me last[level]=" +
                                    (theToken.last[toke.tokenLevel] == null ? "null" : theToken.last[toke.tokenLevel].ToString());
                            Isis.WriteLine(because);
                        }
                        valid = false;
                    }
                    else
                        theToken.lastValidated[toke.tokenLevel] = true;
                    if (valid)
                    {
                        if (gcollectedTo > theView.NextIncomingMsgID[1])
                            throw new IsisException("<" + gname + "> has already garbage collected to " + gcollectedTo + ", but I'm waiting for " + theView.NextIncomingMsgID[1] + ", with stableAtSender=" + theToken.stableAtSender + IsisSystem.GetState());

                        if (toke.gaddr.Equals(Isis.ISISMEMBERS.gaddr) && Isis.ClientOf != null && !Isis.ClientOf.Equals(toke.groupOwner))
                        {
                            becomeClientOf(toke.groupOwner);
                            if (Isis.ClientOf.Equals(toke.groupOwner))
                                theToken.groupOwner = toke.groupOwner;
                            watchingForStableTo = true;
                        }

                        if (switchOwner)
                            switchToNewOwner(toke);

                        if (toke.logicalClock > theToken.logicalClock)
                        {
                            theToken.logicalClock = toke.logicalClock;
                            theToken.resetStableByLevel(this);
                            theToken.state = toke.state;
                            int vid = theToken.IAmLgOwner ? theToken.theGroup.myFirstLeadershipView : theToken.WorkingView.viewid;
                            switch (theToken.state)
                            {
                                case tokenInfo.NORMAL:
                                    break;
                                case tokenInfo.INQUIRY:
                                    if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                                        Isis.WriteLine("Setting Aggregator Value(INQUIRY in vid=" + vid + "): " + theToken.groupOwner + ":" + tokenInfo.INQUIRY + " to true ");
                                    SetAggregatorValueAndVID<FlushAggKey, bool>(vid, new FlushAggKey(theToken.groupOwner, tokenInfo.INQUIRY), true);
                                    break;
                                case tokenInfo.SETSTABLETO:
                                    if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                                        Isis.WriteLine("Setting Aggregator Value(STABLETO in vid=" + vid + "): " + theToken.groupOwner + ":" + tokenInfo.SETSTABLETO + " to true ");
                                    SetAggregatorValueAndVID<FlushAggKey, bool>(vid, new FlushAggKey(theToken.groupOwner, tokenInfo.SETSTABLETO), true);
                                    break;
                            }
                        }

                        byte[][] bvs = toke.incomingValuesArray;
                        if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                        {
                            Isis.WriteLine("Aggregation logic: received bvs[" + bvs.Length + "][], list of IAggregateEventHandlers: ");
                            for (int i = 0; i < bvs.Length; i++)
                                Isis.WriteLine("    bvs[" + i + "]: byte vector of length " + bvs[i].Length);
                            foreach (IAggregateEventHandler ae in theToken.theGroup.AggList[toke.tokenLevel])
                                Isis.WriteLine("    ae=" + ae.GetType().ToString());
                        }
                        if (theToken.myOffset[toke.tokenLevel] == 0)
                        {
                            // 0'th ranked guy gets incoming token: treat this as a message up from toke.tokenLevel to next higher level
                            if (toke.tokenLevel < theToken.nlevels - 1)
                            {
                                // Inner level of the tree
                                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                                    Isis.WriteLine("Aggregation logic: Inner level of the tree and I am at rank0 for next level (calling GotDValue)");
                                int idx = 0;
                                foreach (IAggregateEventHandler ae in theToken.theGroup.AggList[toke.tokenLevel + 1])
                                {
                                    object[] KeysNValues = Msg.BArrayToObjects(bvs[idx++]);
                                    int len = (int)(KeysNValues[Group.LEN]);
                                    int[] vids = (int[])(KeysNValues[Group.VIDS]);
                                    for (int n = 0; n < len; n++)
                                        ae.AggEvent(Group.GotDValue, vids[n], KeysNValues[Group.KEYS], KeysNValues[Group.VALS], n);
                                }
                            }
                            else
                            {
                                // Root level: Computed the aggregated answer
                                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                                    Isis.WriteLine("Aggregation logic: Root level of the tree and I am at rank0 (calling GotAggRes)");
                                int idx = 0;
                                foreach (IAggregateEventHandler ae in theToken.theGroup.AggList[toke.tokenLevel])
                                {
                                    object[] KeysNValues = Msg.BArrayToObjects(bvs[idx++]);
                                    int len = (int)(KeysNValues[Group.LEN]);
                                    int[] vids = (int[])(KeysNValues[Group.VIDS]);
                                    for (int n = 0; n < len; n++)
                                        ae.AggEvent(Group.GotAggRes, vids[n], KeysNValues[Group.KEYS], KeysNValues[Group.VALS], n);
                                }
                            }
                        }
                        else
                        {
                            // i'th ranked guy gets incoming token from i-1'th ranked guy
                            if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                                Isis.WriteLine("Aggregation logic: Received from left, will pass to my right (calling GotRValue)");
                            int idx = 0;
                            foreach (IAggregateEventHandler ae in theToken.theGroup.AggList[toke.tokenLevel])
                            {
                                object[] KeysNValues = Msg.BArrayToObjects(bvs[idx++]);
                                int len = (int)(KeysNValues[Group.LEN]);
                                int[] vids = (int[])(KeysNValues[Group.VIDS]);
                                for (int n = 0; n < len; n++)
                                    ae.AggEvent(Group.GotRValue, vids[n], KeysNValues[Group.KEYS], KeysNValues[Group.VALS], n);
                            }
                        }

                        theToken.lastToken[toke.tokenLevel] = toke;
                        if (theToken.inhibitResenderLoop)
                        {
                            bool GotAll = true;
                            for (int i = 0; i < theToken.nlevels; i++)
                                if (theToken.last[i] != null && !theToken.last[i].isMyAddress() && (theToken.lastToken[i] == null || theToken.lastToken[i].logicalClock != theToken.logicalClock))
                                    GotAll = false;
                            if (GotAll && theToken.gotAllAt == 0)
                                theToken.gotAllAt = Isis.NOW();
                            if (theToken.gotAllAt > 0 && (Isis.NOW() - theToken.gotAllAt > Isis.ISIS_DEFAULTTIMEOUT * 2))
                                theToken.inhibitResenderLoop = false;
                        }

                        using (new LockAndElevate(theToken.slock))
                        {
                            theToken.stableTo = Math.Max(toke.stableTo, theToken.stableTo);
                            theToken.stableVID = Math.Max(toke.stableVID, theToken.stableVID);
                            if (theToken.stableTo > theToken.unstableVIDMID && theToken.unstableVIDMID > 0)
                            {
                                theToken.stableVID = theToken.unstableVID;
                                if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENFLUSH)) != 0)
                                    Isis.WriteLine("New value of theToken.stableVID: " + theToken.stableVID);
                                theToken.unstableVIDMID = 0;

                            }
                            if (theToken.viewDeltas != null)
                                theToken.fixVDS();
                        }
                    }
                    using (new LockAndElevate(UnstableLock))
                    {
                        List<Msg> newUnstable;
                        if (valid && watchingForStableTo && toke.stableTo != -1)
                        {
                            // new group owner is active now and has announced the stableTo value that ends the multicasts from the prior sender
                            // Delete any messages that have larger msgid's and came from the old sender: there must have been a gap in the
                            // sequence and they are orphans (dangerous ones because, if delivered, they would cause confusion!)
                            gcollectedTo = toke.stableTo;
                            newUnstable = new List<Msg>();
                            foreach (Msg m in Unstable)
                                if (m.msgid > toke.stableTo && m.sender.Equals(toke.groupOwner))
                                    newUnstable.Add(m);
                            Unstable = newUnstable;
                        }
                    }
                    if (valid)
                    {
                        updateSeenInfo();

                        // This is used in the token thread, but only by the "point guy" for each ring
                        bool hasBelow = true;
                        for (int level = 0; level < toke.tokenLevel; level++)
                            if (theToken.StableByLevel[level] == -1)
                            {
                                hasBelow = false;
                                break;
                            }
                        if (hasBelow)
                        {
                            int min = toke.aggStable;
                            if (toke.tokenLevel > 0)
                                min = Math.Min(min, theToken.StableByLevel[toke.tokenLevel - 1]);
                            if ((IsisSystem.Debug & IsisSystem.TOKENSTABILITY) != 0)
                                Isis.WriteLine("<" + gname + ">: LogicalClock=" + toke.logicalClock + ": Decided to set theToken.StableByLevel[" + toke.tokenLevel + "]=" + min +
                                    "(token is from " + toke.sender + ", has logical clock=" + toke.logicalClock + ", has level=" + toke.tokenLevel + ", has aggstable=" + toke.aggStable + ")");
                            theToken.StableByLevel[toke.tokenLevel] = min;
                        }
                        if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                            Isis.WriteLine("After processIncoming incoming token: my " + theToken);
                    }
                }
            }
            catch (Exception e)
            {
                Isis.WriteLine("processIncoming threw an exception! " + e);
            }
        }

        internal void gotNewViewDeltas(Isis.ViewDelta[] vds)
        {
            tokenInfo theToken;
            View theView;
            using (new LockAndElevate(TokenLock))
            using (new LockAndElevate(ViewLock))
            {
                theToken = this.theToken;
                theView = this.theView;
            }
            vds = vds.Select(vd => vd).Where(vd => vd.gaddr.Equals(this.gaddr) && vd.prevVid >= (theView == null ? -1 : theView.viewid)).ToArray();
            if (vds.Length == 0)
                return;
            Group tg = doLookup(gaddr);
            if (tg != this)
            {
                // Use the true group, not the proxy...
                if (tg != null)
                    tg.gotNewViewDeltas(vds);
                return;
            }
            if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
            {
                Isis.WriteLine("Token layer got new view deltas:");
                foreach (Isis.ViewDelta vd in vds)
                    Isis.WriteLine("  " + vd);
            }
            if (theToken != null)
            {
                int vid = theView.viewid;
                foreach (Isis.ViewDelta vd in theToken.viewDeltas)
                    if (vd.prevVid == vid)
                        ++vid;
                foreach (Isis.ViewDelta vd in vds)
                    if (vd.prevVid < vid)
                        continue;
                    else if (vd.prevVid > vid++)
                        return;
                theToken.applyViewDeltas(this, vds);
            }
            if (Isis.ISIS_TCP_ONLY && this == Isis.ISISMEMBERS)
                ReliableSender.TCPNoteVDS(vds);
        }

        internal void switchToNewOwner(tokenInfo toke)
        {
            tokenInfo theToken;
            using (new LockAndElevate(TokenLock))
                theToken = this.theToken;
            int oldOwnerRank = theToken.WorkingView.GetRankOf(theToken.groupOwner);
            int newOwnerRank = theToken.WorkingView.GetRankOf(toke.groupOwner);
            int r;
            if (oldOwnerRank >= newOwnerRank)
                return;
            for (r = 0; r < theToken.WorkingView.members.Length; r++)
                if (!theView.hasFailed[r])
                    break;
            if (r != newOwnerRank)
            {
                if ((IsisSystem.Debug & IsisSystem.FAILURES) != 0)
                    Isis.WriteLine("WARNING: switch to New Owner but he doesn't seem to be the owner!  I compute rank " + r + ", he asserts rank " + newOwnerRank + "  (calling note-failed on lower-ranked procs)");
                while (r != newOwnerRank)
                    View.noteFailed(this, theToken.WorkingView.members[r++]);
            }
            theToken.groupOwner = toke.groupOwner;
            ReliableSender.resetTheToken(this);
        }

        internal static LockObject flock = new LockObject("token.flock");

        private static void ForwardMsg(tokenInfo toke, List<ReliableSender.MsgDesc> mds)
        {
            if (mds.Count() == 0)
                return;
            int cnt = 0;
            using (new LockAndElevate(flock))
                foreach (ReliableSender.MsgDesc md in mds)
                    if (ForwardFromPSB(toke, md))
                        ++cnt;
            using (new LockAndElevate(IsisSystem.RTS.Lock))
                IsisSystem.RTS.TTRet += cnt;
        }

        private static bool ForwardMsg(tokenInfo toke, Msg m)
        {
            if (m.wasForwardedTo != null && m.wasForwardedTo.Equals(toke.sender))
                return false;
            int delta = m.msgid - toke.alsoSeenBase - 1;
            if (delta >= 0 && delta < 64 && (toke.alsoSeen & (1L << delta)) == 0)
            {
                if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.NACKS)) != 0)
                    Isis.WriteLine("[" + Isis.TimeToString(Isis.NOW()) + "]: Token-based recovery: forwarding unstable msg " + m.vid + ":" + m.msgid + " to " + toke.sender +
                        " because delta=" + delta + " and alsoSeenBase=" + toke.alsoSeenBase + " and  alsoseen=" + toke.alsoSeen.ToString("X"));
                m.wasForwardedTo = toke.sender;
                m.gaddr = toke.gaddr;
                ReliableSender.Forward(toke.sender, toke.gaddr, ReliableSender.ResendForm(m));
                return true;
            }
            return false;
        }

        private static bool ForwardFromPSB(tokenInfo toke, ReliableSender.MsgDesc md)
        {
            if (Isis.NOW() < md.resendTime)
                return false;
            int delta = md.MsgId - toke.alsoSeenBase - 1;
            if (delta >= 0 && delta < 64 && (toke.alsoSeen & (1L << delta)) == 0)
            {
                if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.NACKS)) != 0)
                    Isis.WriteLine("[" + Isis.TimeToString(Isis.NOW()) + "]: Token-based recovery: forwarding unstable msg from lg-pendingsend-buffer " + md.MsgVid + ":" + md.MsgId + " to " + toke.sender +
                        " because delta=" + delta + " and alsoSeenBase=" + toke.alsoSeenBase + " and  alsoseen=" + toke.alsoSeen.ToString("X") + " and was last forwarded at " + Isis.TimeToString(md.resendTime));
                md.resendTime = Isis.NOW() + 500;
                ReliableSender.ResendP2P(toke.sender, md);
                return true;
            }
            return false;
        }

        // In a large group, on learning of an unstable message, check to see if prior ring member lacked it
        // If so, forward it.  Also, check and forward when token arrives and I have something prior guy lacked
        internal void checkLastToken(Msg m)
        {
            tokenInfo theToken;
            using (new LockAndElevate(TokenLock))
                theToken = this.theToken;
            List<tokenInfo> ForwardTo = new List<tokenInfo>();
            using (new LockAndElevate(UnstableLock))
            {
                if (theToken == null)
                    return;
                for (int level = 0; level < theToken.nlevels; level++)
                    if (theToken.lastToken[level] != null)
                        ForwardTo.Add(theToken.lastToken[level]);
            }
            if (ForwardTo.Count() > 0)
                foreach (tokenInfo ti in ForwardTo)
                    ForwardMsg(ti, m);
        }

        internal LockObject GotAMsgLock = new LockObject("GotAMsgLock", ThreadPriority.Highest);

        internal bool GotAMsg(Msg m, byte type, string where)
        {
            m.toDoTime = Isis.NOW();
            bool rval;
            using (new LockAndElevate(GotAMsgLock))
                try
                {
                    rval = _GotAMsg(m, type, where);
                }
                catch (IsisException e)
                {
                    if ((!GroupOpen && WasOpen) || (!IsisSystem.IsisActive && IsisSystem.IsisWasActive))
                        return false;
                    throw new IsisException("Uncaught IsisException in group <" + gname + ">: " + e.Message);
                }
            using (new LockAndElevate(ToDoLock))
                if (ToDo.Count() > 1000)
                    throw new IsisException("After GotAMsg ToDo list had > 1000 msgs on it");
            return rval;
        }

        internal bool _GotAMsg(Msg m, byte type, string where)
        {
            List<Msg> willDeliver = new List<Msg>();

            using (new LockAndElevate(PendingQueueLock))
            {
                using (new LockAndElevate(ViewLock))
                {
                    int which = -1, vid = m.vid - 2;
                    if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                        using (new LockAndElevate(ReliableSender.ackInfoLock))
                            ReliableSender.ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: GotAMsg(type=" + type + "): " + m + "\r\n");
                    if (theView != null)
                    {
                        if ((flags & G_ISLARGE) != 0)
                            // In a large group, there is only one legal sender
                            which = 0;
                        else if ((m.flags & Msg.SENTBYORACLE) != 0 && this != Isis.ORACLE)
                            which = -1;
                        else
                            which = theView.GetRankOf(m.sender);
                        vid = theView.viewid;
                        if (which == -1 && theView.currentOracleRep.Equals(m.sender) == false)
                        {
                            theView.currentOracleRep = m.sender;
                            theView.NextIncomingMsgID[0] = 0;
                        }
                        if ((flags & G_ISLARGE) != 0 && m.offWire != null && !theView.IAmLeader() && theToken != null && m.msgid > theToken.stableTo)
                            ReliableSender.LgAdd(this, m, type);
                    }
                    if ((IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0 || (flags & G_TRACE) != 0)
                        Isis.WriteLine("GotAMsg<" + gname + ">: event type=" + Msg.mtypes[type] + ", sender " + m.sender + " rank is " + which + ", m.offWire is " + (m.offWire != null ? "non" : "") + "null\r\n   " + m);
                    if ((!GroupOpen && WasOpen) || (theView != null && theView.members.Length == 0 && theView.leavers.Contains(Isis.my_address)))
                        return true;
                    m.offWire = null;
                    if (gname.Equals("ORACLE"))
                    {
                        if (which == -1 && !m.sender.isMyAddress())
                        {
                            if (theView != null)
                            {
                                if ((IsisSystem.Debug & IsisSystem.DELIVERY) != 0)
                                    Isis.WriteLine("<" + gname + ">: GotAMsg/ORACLE delivering " + m);
                                goto doOracleDelivery;
                            }
                            else if (Isis.my_address.CompareTo(m.sender) < 0 && m.vid == 0 && m.msgid == -1)
                            {
                                if ((IsisSystem.Debug & IsisSystem.STARTSEQ) != 0)
                                    Isis.WriteLine("GotAMsg in ORACLE, sending a message to inhibit creation of new ORACLEs to " + m.sender);
                                goto doOraclePureP2PSend;
                            }
                            else if (m.vid > 0 && m.msgid >= 0)
                                // Arises when a message to v:id arrives in a joining process before initial view for ORACLE was seen
                                using (new LockAndElevate(ToDoLock))
                                    ToDo.Add(m);
                            return true;
                        }
                        else if (m.vid == 0 && m.msgid == -1)
                        {
                            if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.DISCARDS)) != 0)
                                Isis.WriteLine("WARNING: ISIS ignoring a duplicate ORACLE.JOIN: " + m);
                            using (new LockAndElevate(IsisSystem.RTS.Lock))
                                IsisSystem.RTS.Discarded++;
                            return true;
                        }
                    }

                    bool needsxfer;
                    using (new LockAndElevate(GroupFlagsLock))
                        needsxfer = (flags & G_NEEDSTATEXFER) != 0 && (m.flags & Msg.SENTBYORACLE) == 0;
                    if (!HasFirstView || (needsxfer && m.vid > 0))
                    {
                        List<Msg> newToDo = new List<Msg>();
                        using (new LockAndElevate(ToDoLock))
                        {
                            foreach (Msg lm in ToDo)
                                if (m == null)
                                    newToDo.Add(lm);
                                else if (lm.Equals(m))
                                {
                                    if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.DISCARDS)) != 0)
                                        Isis.WriteLine("WARNING: ISIS ignoring a duplicate multicast already on ToDo list -- case 1 (" + m.sender + " sent to " + m.vid + ":" + m.msgid + ")");
                                    using (new LockAndElevate(IsisSystem.RTS.Lock))
                                        IsisSystem.RTS.Discarded++;
                                    return true;
                                }
                                else
                                {
                                    if (lm.vid > m.vid || lm.msgid > m.msgid)
                                    {
                                        newToDo.Add(m);
                                        m = null;
                                    }
                                    newToDo.Add(lm);
                                }
                            if (m != null)
                                newToDo.Add(m);
                            ToDo = newToDo;
                        }
                        return true;
                    }
                    if (m.vid == 0)
                        using (new LockAndElevate(GroupFlagsLock))
                        {
                            if ((flags & G_NEEDSTATEXFER) != 0)
                                xferWait.Release();
                            flags &= ~G_NEEDSTATEXFER;
                        }
                    if (m.vid != vid && (flags & G_ISLARGE) == 0 && (m.flags & Msg.SENTBYORACLE) == 0)
                    {
                        if (m.vid < vid)
                        {
                            if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.DISCARDS)) != 0)
                                Isis.WriteLine("WARNING: ISIS ignoring a multicast to an old view (" + m.sender + " sent to " + m.vid + ":" + m.msgid + " but current viewid is " + vid + ")" + m);
                            using (new LockAndElevate(IsisSystem.RTS.Lock))
                                IsisSystem.RTS.Discarded++;
                            return true;
                        }
                        using (new LockAndElevate(ToDoLock))
                        {
                            List<Msg> newToDo = new List<Msg>();
                            foreach (Msg lm in ToDo)
                                if (m == null)
                                    newToDo.Add(lm);
                                else if (lm.Equals(m))
                                {
                                    if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.DISCARDS)) != 0)
                                        Isis.WriteLine("WARNING: ISIS ignoring a multicast already on ToDo list -- case 2 (" + m.sender + " sent to " + m.vid + ":" + m.msgid + ")");
                                    using (new LockAndElevate(IsisSystem.RTS.Lock))
                                        IsisSystem.RTS.Discarded++;
                                    return true;
                                }
                                else
                                {
                                    if (m != null && (lm.vid > m.vid || (lm.vid == m.vid && lm.msgid > m.msgid)))
                                    {
                                        newToDo.Add(m);
                                        m = null;
                                    }
                                    newToDo.Add(lm);
                                }
                            if (m != null)
                                newToDo.Add(m);
                            ToDo = newToDo;
                        }
                        return true;
                    }
                    if (PendingQueue == null)
                    {
                        if ((flags & G_ISLARGE) != 0)
                        {
                            if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                                using (new LockAndElevate(ReliableSender.ackInfoLock))
                                    ReliableSender.ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: GotAMsg -- willDeliver(1) " + m.sender + "::" + m.vid + ":" + m.msgid + "\r\n");
                            willDeliver.Add(m);
                            goto do_delivery;
                        }
                        throw new IsisException("*** WARNING: In GotAMsg but PendingQueue for<" + gname + "> is null!");
                    }
                    SortedList<long, Msg> MsgQ;
                    MsgQ = PendingQueue[1 + which];
                    if (MsgQ == null)
                    {
                        if (m.vid == vid + 1 && m.msgid == 0)
                        {
                            willDeliver.Add(m);
                            goto do_delivery;
                        }
                        if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.DISCARDS)) != 0)
                            Isis.WriteLine("WARNING: ISIS ignoring an unexpected multicast[PendingQueue null] (" + m.sender + "sent to " + m.vid + ":" + m.msgid + ", but NextIncomingMsgID is " + vid + ":" + theView.NextIncomingMsgID[1 + which] + ").... " + m);
                        using (new LockAndElevate(IsisSystem.RTS.Lock))
                            IsisSystem.RTS.Discarded++;
                        return m.vid < vid;
                    }
                    bool foundKey = false;
                    foreach (KeyValuePair<long, Msg> kvp in MsgQ)
                        if (kvp.Key == m.key(flags))
                        {
                            Msg lm = kvp.Value;
                            if (lm.vid != m.vid)
                                throw new IsisException("PendingQueue[" + which + "] contained an entry for " + lm + " that confused me (wrong vid) when trying to insert " + m);
                            foundKey = true;
                            break;
                        }
                    if (!foundKey)
                    {
                        MsgQ.Add(m.key(flags), m);
                        if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                            using (new LockAndElevate(ReliableSender.ackInfoLock))
                                ReliableSender.ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: GotAMsg added to MsgQ[" + (1 + which) + "], new count=" + MsgQ.Count() + "\r\n");
                    }
                    else
                    {
                        if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.DISCARDS)) != 0)
                        {
                            string Known = " ";
                            foreach (KeyValuePair<long, Msg> kvp in MsgQ)
                                Known += "(Key=" + (kvp.Key >> 32) + ":" + (kvp.Key & 0xFFFFFFFF) + ", Value=" + kvp.Value.sender + "::" + kvp.Value.vid + ":" + kvp.Value.msgid + ")";
                            Isis.WriteLine("WARNING: ISIS ignoring a duplicate multicast[2] (" + m.sender + " sent to " + m.vid + ":" + m.msgid +
                                ", but NextIncomingMsgID is " + vid + ":" + theView.NextIncomingMsgID[1 + which] + "); known = " + Known);
                        }
                        using (new LockAndElevate(IsisSystem.RTS.Lock))
                            IsisSystem.RTS.Discarded++;
                        return true;
                    }
                    if ((flags & G_ISLARGE) == 0)
                    {
                        Msg nxtM = null;
                        int rcnt = MsgQ.Count();
                        while (rcnt-- > 0 && (nxtM = MsgQ.ElementAt(0).Value) != null && (nxtM.vid < vid || (nxtM.vid == vid && nxtM.msgid < theView.NextIncomingMsgID[1 + which])))
                        {
                            if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.DISCARDS)) != 0)
                                using (new LockAndElevate(IsisSystem.RTS.Lock))
                                    IsisSystem.RTS.Discarded++;
                            MsgQ.RemoveAt(0);
                            if (which != -1 && !theView.members[which].Equals(nxtM.sender))
                                throw new IsisException("MsgQ was for sender " + theView.members[1 + which] + " (which=" + which + "), but found a message on it from " + nxtM.sender);
                            else if (which == -1 && (nxtM.flags & Msg.SENTBYORACLE) == 0)
                                throw new IsisException("MsgQ was for ORACLE, but found a message on it from " + nxtM.sender + " with SENTBYORACLE=false");
                            if ((IsisSystem.Debug & IsisSystem.MSGQS) != 0)
                                Isis.WriteLine("<" + gname + ">: DISCARD IN GOTMSG FOR " + nxtM.sender + "::" + nxtM.vid + ":" + nxtM.msgid + (((nxtM.flags & Msg.SENTBYORACLE) != 0) ? " * sent by Oracle" : "") +
                                    " because which=" + which + "and theView.NextIncomingMsgID[1+which]=" + theView.NextIncomingMsgID[1 + which]);
                            nxtM = null;
                        }
                        if (nxtM != null)
                        {
                            if (nxtM.vid == vid && ((nxtM.nRaw > 0 && nxtM.msgid - nxtM.nRaw <= theView.NextIncomingMsgID[1 + which]) || nxtM.msgid == theView.NextIncomingMsgID[1 + which]))
                            {
                                Address sendTo;
                                if ((flags & G_ISLARGE) == 0 || nxtM.msgid == theView.NextIncomingMsgID[1 + which] + 1)
                                    sendTo = theView.members[(theView.GetMyRank() + 1) % theView.members.Length];
                                else
                                    sendTo = theView.members[0];
                                ReliableSender.SendNack(this, sendTo, nxtM.sender, vid, theView.NextIncomingMsgID[1 + which]);
                            }
                        }
                    }
                    if (MsgQ.Count() > 0)
                        DequeueDeliverableMsgs(willDeliver, which, theView, false);
                }

            do_delivery:
                while (willDeliver.Count() > 0)
                {
                    foreach (Msg dm in willDeliver)
                    {
                        if ((IsisSystem.Debug & IsisSystem.MSGQS) != 0)
                            Isis.WriteLine("<" + gname + ">: incomingSends.put(" + dm.sender + "::" + dm.vid + ":" + dm.msgid);
                        incomingSends.put(dm);
                    }
                    willDeliver = new List<Msg>();
                    using (new LockAndElevate(ViewLock))
                    {
                        for (int w = 0; w < PendingQueue.Length; w++)
                            if (PendingQueue[w] != null)
                                DequeueDeliverableMsgs(willDeliver, w - 1, theView, false);
                    }
                }
            }
            return true;

        doOracleDelivery:
            doDeliveryCallbacks(m, "GotAMsg - ORACLE/-1 case", type);
            return true;

        doOraclePureP2PSend:
            Isis.ORACLE.doPureP2PSend(m.sender, true, Isis.JOIN, Isis.my_address);
            return true;
        }

        internal void doDelivery(BoundedBuffer from, string which)
        {
            try
            {
                object o;
                while (IsisSystem.IsisActive && (o = from.get()) != null)
                {
                    IsisSystem.RTS.ThreadCntrs[12]++;
                    if (o.GetType().Equals(typeof(Msg)))
                        doMsgDelivery((Msg)o);
                    else if (o.GetType().Equals(typeof(List<Msg>)))
                        foreach (Msg m in (List<Msg>)o)
                            doMsgDelivery(m);
                    else if (o.GetType().Equals(typeof(View)))
                    {
                        View v = (View)o;
                        if ((IsisSystem.Debug & IsisSystem.MSGQS) != 0 && this != Isis.ORACLE && this != Isis.ISISMEMBERS)
                            Isis.WriteLine("<" + gname + ">: DELIVERY VIEW " + v.viewid + ", " + v.members.Length + " members");
                        if (v.theChkptMakers != null)
                            runChkptMakers(v);
                        List<VHCallBack> vhcbl = new List<VHCallBack>();
                        if (from == incomingSends)
                        {
                            using (new LockAndElevate(ViewHandlers.vhListLock))
                                foreach (VHCallBack vhcb in ViewHandlers.vhList)
                                    vhcbl.Add(vhcb);
                            foreach (VHCallBack vhcb in vhcbl)
                                if (vhcb.withLock)
                                {
                                    VHCallBack myVhcb = vhcb;
                                    new Thread(delegate()
                                    {
                                        try
                                        {
                                            Thread.CurrentThread.Name = "Callback thread for viewhander in <" + gname + ">";
                                            using (new ILock(ILock.LLENTRY, gaddr))
                                                myVhcb.vhProc(v);
                                        }
                                        catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                                        IsisSystem.ThreadTerminationMagic();
                                    }).Start();
                                }
                                else
                                    vhcb.vhProc(v);
                        }
                        if (v.members.Length == 0 && v.leavers.Length == 1 && v.leavers[0].isMyAddress())
                            return;
                        CallbacksDone = true;
                    }
                    else
                        throw new IsisException("doDelivery: didn't expect to find object of type " + o.GetType());
                }
                BoundedBuffer.unregister(from);
                if (IsisSystem.IsisActive && GroupOpen)
                    throw new IsisException("doDelivery<" + gname + ">");
            }
            catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
        }

        private void doMsgDelivery(Msg m)
        {
            if (((IsisSystem.Debug & IsisSystem.MSGQS) != 0 && this != Isis.ORACLE && this != Isis.ISISMEMBERS) || (IsisSystem.Debug & IsisSystem.DELIVERY) != 0)
                Isis.WriteLine("<" + gname + ">: DELIVERY " + m.sender + "::" + m.vid + ":" + m.msgid + (((m.flags & Msg.SENTBYORACLE) != 0) ? " * sent by Oracle" : ""));
            if (m.type == Msg.ISGRPP2P || m.type == Msg.ISRAWGRPP2P)
                doAction(m);
            else
                doDeliveryCallbacks(m, "GotAMsg/From " + m.sender, m.type);
        }

        private void runChkptMakers(View v)
        {
            if (v.joiners.Length == 0 && !isPersistent())
                return;
            nextView = v;
            inhibitEOC = true;
            if (v.theChkptMakers.Count() > 0)
                foreach (ChkptMaker cpm in v.theChkptMakers)
                    cpm(v);
            inhibitEOC = false;
            EndOfChkpt();
            nextView = null;
        }

        private void doAction(Msg m)
        {
            if ((m.flags & Msg.FRAGGED) != 0)
            {
                Group.deFragRdv(this, m);
                return;
            }
            object[] obs = Msg.MsgToObjects(m);
            if (obs[0].GetType().Equals(typeof(Msg)))
            {
                m = (Msg)obs[0];
                obs = Msg.MsgToObjects(m);
            }
            setReplyTo(m);
            cbAction(m, m.vid, m.msgid, m.sender, obs);
            clearReplyTo();
        }

        internal static int Undelivered()
        {
            int cnt = 0;
            List<Group> gl = IsisGroupsClone();
            foreach (Group g in gl)
            {
                if (!g.HasFirstView)
                    continue;
                using (new LockAndElevate(g.PendingQueueLock))
                {
                    for (int i = 0; i < g.PendingQueue.Length; i++)
                        if (g.PendingQueue[i] != null)
                            cnt += g.PendingQueue[i].Count();
                }
            }
            return cnt;
        }

        // Must be called while holding a lock on PendingQueue
        private void DequeueDeliverableMsgs(List<Msg> dq, int which, View v, bool informFC)
        {
            Msg m = null;
            SortedList<long, Msg> MsgQ = PendingQueue[1 + which];
            if (MsgQ == null)
                return;
            if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                using (new LockAndElevate(ReliableSender.ackInfoLock))
                    ReliableSender.ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: DequeueDeliverableMsgs -- MsgQ count=" + MsgQ.Count() + "\r\n");
            int rcnt = MsgQ.Count();
            while (rcnt-- > 0 && (m = MsgQ.ElementAt(0).Value) != null && m.vid <= v.viewid && m.msgid - m.nRaw <= v.NextIncomingMsgID[1 + which])
            {
                MsgQ.RemoveAt(0);
                if ((m.nRaw > 0 & m.msgid - m.nRaw >= v.NextIncomingMsgID[1 + which]) || m.msgid == v.NextIncomingMsgID[1 + which])
                {
                    if (m.vid != v.viewid && (flags & G_ISLARGE) == 0)
                    {
                        if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.DISCARDS)) != 0)
                            Isis.WriteLine("WARNING<" + gname + ">: ISIS ignoring a duplicate multicast[3] (" + m.sender + " sent to " + m.vid + ":" + m.msgid + ", but NextIncomingMsgID is " + theView.viewid + ":" + theView.NextIncomingMsgID[1 + which] + ")");
                        using (new LockAndElevate(IsisSystem.RTS.Lock))
                            IsisSystem.RTS.Discarded++;
                    }
                    else
                    {
                        v.NextIncomingMsgID[1 + which] = m.msgid + 1;
                        if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                            using (new LockAndElevate(ReliableSender.ackInfoLock))
                                ReliableSender.ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: DequeueDeliverableMsgs -- willDeliver " + m.sender + "::" + m.vid + ":" + m.msgid + "\r\n");
                        dq.Add(m);
                        if ((IsisSystem.Debug & IsisSystem.MSGQS) != 0 && this != Isis.ORACLE && this != Isis.ISISMEMBERS)
                            Isis.WriteLine("< " + gname + ">: DQ " + m.sender + "::" + m.vid + ":" + m.msgid + (((m.flags & Msg.SENTBYORACLE) != 0) ? " * sent by Oracle" : ""));
                        if (which != -1 && v.LastMsg != null && v.NextIncomingMsgID[1 + which] == v.LastMsg[which])
                        {
                            PendingQueue[1 + which] = null;
                            break;
                        }
                    }
                }
                else
                {
                    if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                        Isis.WriteLine("WARNING: in DequeueDeliverableMsgs<" + gname + ">, unexpectedly dequeued message with " +
                                m.sender + " sent to " + m.vid + ":" + m.msgid + (((m.flags & Msg.SENTBYORACLE) != 0) ? " * sent by Oracle" : "") + ", yet expected msgid in the range " +
                                theView.viewid + ":[" + v.NextIncomingMsgID[1 + which] + "-" + (v.NextIncomingMsgID[1 + which] + m.nRaw) + "]");
                    if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                        using (new LockAndElevate(ReliableSender.ackInfoLock))
                            ReliableSender.ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: DequeueDeliverableMsgs, unexpectedly dequeued message with msgid=" +
                                m.sender + " sent to " + m.vid + ":" + m.msgid + ", yet expected " + theView.viewid + ":" + v.NextIncomingMsgID[1 + which] + "\r\n");
                }
            }
            if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                using (new LockAndElevate(ReliableSender.ackInfoLock))
                    if (m != null)
                        ReliableSender.ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: DequeueDeliverableMsgs -- enqueued for later delivery " + m.sender + "::" + m.vid + ":" + m.msgid + "\r\n");
                    else
                        ReliableSender.ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: DequeueDeliverableMsgs -- delivery finished (MsgQ[" + (1 + which) + "].count=" + MsgQ.Count() + "... queue is empty)\r\n");
        }

        private LockObject LockIt = new LockObject("LockIt");
        private Dictionary<int, Msg> replyMsgs = new Dictionary<int, Msg>(16);

        internal void setReplyTo(Msg m)
        {
            setReplyTo(m, Thread.CurrentThread);
        }

        internal void setReplyTo(Msg m, Thread t)
        {
            //DumpReplyTo("setReplyTo:"+m+"::"+t.ManagedThreadId);
            if ((m.flags & Msg.NEEDSREPLY) == 0)
                return;
            using (new LockAndElevate(LockIt))
            {
                if (replyMsgs.ContainsKey(t.ManagedThreadId))
                    replyMsgs[t.ManagedThreadId] = m;
                else
                    replyMsgs.Add(t.ManagedThreadId, m);
            }
        }

        internal void clearReplyTo()
        {
            //DumpReplyTo("clearReplyTo");
            using (new LockAndElevate(LockIt))
            {
                if (replyMsgs.ContainsKey(Thread.CurrentThread.ManagedThreadId))
                    replyMsgs.Remove(Thread.CurrentThread.ManagedThreadId);
            }
        }

        internal Msg getReplyTo()
        {
            //DumpReplyTo("getReplyTo::" + Thread.CurrentThread.ManagedThreadId);
            Msg rmsg = null;
            using (new LockAndElevate(LockIt))
                replyMsgs.TryGetValue(Thread.CurrentThread.ManagedThreadId, out rmsg);
            return rmsg;
        }

        internal Msg getReplyToAndClear()
        {
            //DumpReplyTo("getReplyToAndClear::" + Thread.CurrentThread.ManagedThreadId);
            Msg rmsg = null;
            using (new LockAndElevate(LockIt))
                if (replyMsgs.TryGetValue(Thread.CurrentThread.ManagedThreadId, out rmsg))
                    replyMsgs.Remove(Thread.CurrentThread.ManagedThreadId);
            return rmsg;
        }

        internal void DumpReplyTo(string where)
        {
            using (new LockAndElevate(LockIt))
            {
                string s = " ";
                foreach (KeyValuePair<int, Msg> kvp in replyMsgs)
                    s += "{" + kvp.Key + "::" + kvp.Value + "}";
                Isis.WriteLine(where + "[" + s + "]");
            }
        }

        /// <summary>
        /// Indicate to Isis that the caller will not reply and asks that Isis NOT send an automated NullReply
        /// </summary>
        /// <remarks>
        /// NoReply() is a risky API that must be used with real care.  We use it ourselves in the SafeSend logic but
        /// SafeSend has a mechanism (part of the new-view flush protocol) to terminate a pending SafeSend if the view changes.
        /// Lacking this, if you call NoReply, the Query sender might still be waiting for a reply from the process that just
        /// said it won't be replying: unlike NullReply, which actually sends a message to the Query sender, NoReply is totally
        /// local.
        /// 
        /// So several things can go wrong.  Suppose the Query asked for ALL replies.  Clearly, it will wedge and eventually
        /// timeout, taking some timeout action against this non-replying process.  Now imagine a Query on a group with N members
        /// thast asks for 1 reply.  Here N-1 members might call NoReply, but what if the 1 member who should send the reply now crashes?
        /// The Query sender will be watching the N-1 other members, and yet will never get the desired reply.  Again, it will timeout
        /// on one or more members.
        /// 
        /// A workaround would run roughly as follows: you could assign the Reply obligation to some thread using SetReplyThread,
        /// which is a bit like NoReply: it inhibits the automatic NullReply that Isis normally sends.  Then, if the designated reply
        /// thread crashes, you can see this via a NewView upcall and then call Reply in that thread.  Otherwise, if (somehow) you can
        /// detect that the Reply was sent safely (e.g. perhaps you get some subsequent multicast and know that the sender must
        /// therefor have received the prior result), you would call NoReply() in your watcher thread, at which point the 
        /// watcher is unlinked from the old message and you can call SetReplyThread again for the new request.  Whew!
        /// </remarks>
        public void NoReply()
        {
            clearReplyTo();
        }

        /// <summary>
        /// Tells Isis that thread other than the one called via a message-delivery upcall will send the reply.
        /// </summary>
        /// <param name="t">The thread that will send the reply (don't call t.start() until after you call SetReplyThread(t)!)</param>
        /// <remarks>
        /// Normally, Isis assumes that the thread handling a Query will send the reply, and if it neglects to do so, sends an automatically-generated NullReply().  A call to SetReplyThread disables
        /// that functionality and tells Isis that the reply will (definitely) be sent by the thread designated by the caller.  
        /// A given thread can only have one message assigned to it using SetReplyThread at a time; attempts to assign two messages
        /// trigger a dictionary exception (two entries with the same key).  
        /// 
        /// A thread becomes unliked from a message by calling Reply(), NullReply(), AbortReply() or NoReply().
        /// 
        /// Note: If you use SetReplyThread(t), and then thread t terminates without sending a reply, the Query caller will
        /// wait until a timeout occurs (perhaps indefinitely if the Query caller used a very big timeout value).  Moreover, Isis will retain a copy of the Query() message
        /// itself, in effect leaking memory.  (The alternative would be for Isis to fork a thread that would issue a t.join(), so as to sense the termination of t,
        /// but the overheads of doing so are too high for this to make sense).
        /// 
        /// Thus, if you elect to use this mechanism, the Query handler should fork a thread, register
        /// it via SetReplyThread, and then start it.  And that child thread MUST remember to call Reply(), NullReply(), NoReply() or AbortReply() before terminating.  
        /// 
        /// The main risk of an error arises in applications
        /// where the Query waits for just 1 reply, sent by "some other process."  In this kind of application, one could seemingly get away with calling SetReplyThread as a way to disable the
        /// mandatory NullReply and in that manner, avoid a coverge-cast at the sender.  This might seem to work, but in fact would be problematic in several ways: first, the Query() message itself will
        /// be retained, which means that such a design would actually leak memory.  Moreover, in a run where failures happen to kill the designated reply
        /// source, the Query caller could hang indefinitely or or timeout, perhaps killing other group members in the process (TO_FAILURE).
        /// </remarks>
        public void SetReplyThread(Thread t)
        {
            Msg rmsg = getReplyToAndClear();
            if (rmsg == null)
                return;
            setReplyTo(rmsg, t);
        }

        internal void doDeliveryCallbacks(Msg m, string where, byte type)
        {
            bool isMulticast = m.dest.Equals(gaddr);
            if (type == Msg.STABILITYINFO)
                return;
            if (m.vid != -1 && m.msgid != -1 && !m.sender.isMyAddress() && (m.flags & Msg.DEFRAGGED) == 0 && m.type != Msg.REDELIVERY)
                using (new LockAndElevate(ViewLock))
                {
                    int rank = -1;
                    if (theView != null)
                        rank = theView.isLarge ? -1 : theView.GetRankOf(m.sender);
                    if (rank != -1 && m.msgid > theView.StableTo[rank + 1])
                        using (new LockAndElevate(UnstableLock))
                            Unstable.Add(m);
                }
            if ((m.flags & Msg.FRAGGED) != 0)
            {
                Group.deFragRdv(this, m);
                return;
            }
            m.type = type;
            m.gaddr = gaddr;
            if ((flags & G_ISLARGE) != 0 && theView != null && theView.nLive() > 2 && m.sender.isMyAddress() == false && m.msgid != -1)
                checkLastToken(m);
            Msg innerMsg;
            object[] obs;
            if (type == Msg.REDELIVERY)
                innerMsg = m;
            else
            {
                obs = Msg.MsgToObjects(m, typeof(Msg));
                innerMsg = (Msg)(obs[0]);
                if (innerMsg.payload == null)
                    innerMsg = m;
            }
            decipherMsg(innerMsg);
            obs = Msg.MsgToObjects(innerMsg);
            if (obs[0].GetType().Equals(typeof(Msg)) && (m.flags & Msg.CIPHER) == 0)
            {
                innerMsg = (Msg)(obs[0]);
                decipherMsg(innerMsg);
                obs = Msg.MsgToObjects(innerMsg);
            }
            isMulticast |= innerMsg.dest.Equals(gaddr);
            CheckForFutureViews(innerMsg);
            if (myLoggingFcn != null && m.Lid != -1)
                myLoggingFcn(IL_DELIVERY, IL_START, m.sender, m.Lid, obs);
            setReplyTo(innerMsg);
            cbAction(innerMsg, innerMsg.vid, innerMsg.msgid, innerMsg.sender, obs);
            if (m.ct != null && durabilityMethod != null)
                durabilityMethod.Done(m.ct);
            if (myLoggingFcn != null && m.Lid != -1)
                myLoggingFcn(IL_DELIVERY, IL_DONE, m.sender, m.Lid, obs);
            clearReplyTo();
        }

        internal void SetCurMsg(Msg m)
        {
            using (new LockAndElevate(curMsgListLock))
            {
                foreach (KeyValuePair<Thread, Msg> kvp in curMsgList)
                    if (kvp.Key.Equals(Thread.CurrentThread))
                    {
                        curMsgList.Remove(kvp);
                        break;
                    }
                if (m != null)
                    curMsgList.Add(new KeyValuePair<Thread, Msg>(Thread.CurrentThread, m));
            }

        }

        /// <summary>
        /// During an upcall, returns the message currently being delivered to the current thread
        /// </summary>
        /// <returns>Current message, or null if none</returns>
        public Msg curMsg()
        {
            using (new LockAndElevate(curMsgListLock))
                foreach (KeyValuePair<Thread, Msg> kvp in curMsgList)
                    if (kvp.Key.Equals(Thread.CurrentThread))
                        return kvp.Value;
            return null;
        }

        /// <summary>
        /// During an upcall, returns the message-id used when the message currently being delivered to the current thread was sent
        /// </summary>
        /// <returns>Current message id, or -1 if none</returns>
        /// <remarks>The message id and viewid are set when the message is sent, and the view may have changed by the time delivery occurs</remarks>
        public int curMsgId()
        {
            using (new LockAndElevate(curMsgListLock))
                foreach (KeyValuePair<Thread, Msg> kvp in curMsgList)
                    if (kvp.Key.Equals(Thread.CurrentThread))
                        return kvp.Value.msgid;
            return Msg.UNINITIALIZED;
        }

        /// <summary>
        /// Useful in C++/CLI
        /// </summary>
        /// <returns>Current group</returns>
        public static Group thisGroup()
        {
            List<Group> clone = IsisGroupsClone();
            foreach (Group g in clone)
                using (new LockAndElevate(g.curMsgListLock))
                    foreach (KeyValuePair<Thread, Msg> kvp in g.curMsgList)
                        if (kvp.Key.Equals(Thread.CurrentThread))
                            return g;
            return null;
        }

        /// <summary>
        /// During an upcall, returns the view-id in which the message currently being delivered to the current thread was sent
        /// </summary>
        /// <returns>View id, or -1 if none</returns>
        /// <remarks>The message id and viewid are set when the message is sent, and the view may have changed by the time delivery occurs</remarks>
        public int curMsgVid()
        {
            using (new LockAndElevate(curMsgListLock))
                foreach (KeyValuePair<Thread, Msg> kvp in curMsgList)
                    if (kvp.Key.Equals(Thread.CurrentThread))
                        return kvp.Value.vid;
            return Msg.UNINITIALIZED;
        }


        internal void enqueueForGroupDelivery(Msg m)
        {
            int retry = 0;
            m.type = Msg.ISGRPP2P;
            while (incomingP2P == null && retry++ < 5)
                Thread.Sleep(500);
            if (incomingP2P == null)
            {
                if (this == Isis.ORACLE && (Isis.ClientOf != null || !Isis.ORACLE.HasFirstView || Isis.ORACLE.theView.GetMyRank() == -1))
                    return;
                throw new IsisException("Unable to enqueue for delivery: p2p message " + m + " in group <" + this.gname + ">");
            }
            else if (Msg.checkView(m))
                incomingP2P.put(m);
            else
                using (new LockAndElevate(P2PStashLock))
                    P2PStash.Add(m);
        }

        internal static void ReportCb(Callable ca, object[] args)
        {
            string sa = " ";
            foreach (object o in args)
                if (o == null)
                    sa += "null, ";
                else
                    sa += "(" + o.GetType() + " = " + o.ToString() + "), ";
            Isis.WriteLine("Callback: " + ca.cb.Method + " with args = (" + sa + ")");
        }

        internal static void ReportCb(Type t, object[] args)
        {
            string sa = " ";
            foreach (object o in args)
                if (o == null)
                    sa += "null, ";
                else
                    sa += "(" + o.GetType() + " = " + o.ToString() + "), ";
            Isis.WriteLine("Calling " + t.Name + ".Constructor(" + sa + ")");
        }

        internal static void ReportCb(Type t, object o)
        {
            string sa = " ";
            if (o == null)
                sa += "null ";
            else
                sa += "(" + o.GetType() + " = " + o.ToString() + ") ";
            Isis.WriteLine("Calling " + t.Name + ".toBArray" + sa);
        }

        private void cbCheck(object[] obs)
        {
            if (isTrackingProxy || isClientProxy)
                return;
            if (obs.Length == 1 && obs[0].GetType().Equals(typeof(object[])))
                obs = (object[])(obs[0]);
            if (obs.Length == 1 && obs[0].GetType().Equals(typeof(Msg)))
                return;
            obs = fixObs(obs);
            List<CallBack> cblist = null;
            int request = rcode(obs);
            if (request == Isis.IM_DHT_GET)
                // Special case for DHTGet
                return;
            if (Handlers[request] == null)
                throw new MissingMemberException("Group doesn't exist, or doesn't allow clients to issue request code " + Isis.rToString(obs[0]));
            cblist = Handlers[request].hList;
            if (cblist != null)
                foreach (CallBack cb in cblist)
                    if (TypeMatch(obs, cb))
                        return;
            if (obs.Length == 1 && obs[0].GetType().Equals(typeof(Msg)))
                return;
            string ts = " ", cbs = " ";
            if (obs.Length == 1)
                ts += "<nil>";
            else for (int i = 1; i < obs.Length; i++)
                    ts += obs[i].GetType() + " ";
            if (cblist != null)
                foreach (CallBack cb in cblist)
                    cbs += "handler[" + tlist(cb) + " ] ";
            throw new MissingMethodException("No handler for request " + Isis.rToString(obs[0]) + " matches provided argument types { " + ts + "} (found " + cbs + ")");
        }

        private static object[] fixObs(object[] obs)
        {
            if (IsSubsetSend(obs[0]))
            {
                object[] newobs = new object[obs.Length - 1];
                for (int n = 1; n < obs.Length; n++)
                    newobs[n - 1] = obs[n];
                obs = newobs;
            }
            return obs;
        }

        private static string tlist(CallBack cb)
        {
            string tls = " ";
            for (int i = 0; i < cb.cbProc.ptypes.Length; i++)
                tls += cb.cbProc.ptypes[i] + " ";
            return tls;
        }

        internal class redeliveryInfo
        {
            internal int recentVid, recentMsgid;
            internal Address recentSender;
            internal Thread recentThread;

            internal redeliveryInfo(int rvid, int rmid, Address rs, Thread rt)
            {
                recentVid = rvid;
                recentMsgid = rmid;
                recentSender = rs;
                recentThread = rt;
            }
        }

        internal LockObject rdiLock = new LockObject("rdiLock");
        internal List<redeliveryInfo> rdiList = new List<redeliveryInfo>();

        private void Redeliver(byte[] barray)
        {
            Redeliver(barray, rdiLock, rdiList);
        }

        private void Redeliver(byte[] barray, LockObject theRdiLock, List<redeliveryInfo> theRdiList)
        {
            redeliveryInfo myRdi = null;
            using (new LockAndElevate(theRdiLock))
            {
                foreach (redeliveryInfo rdi in theRdiList)
                    if (rdi.recentThread == Thread.CurrentThread)
                    {
                        myRdi = rdi;
                        break;
                    }
                if (myRdi != null)
                    theRdiList.Remove(myRdi);
            }
            if (myRdi != null)
                cbAction(null, myRdi.recentVid, myRdi.recentMsgid, myRdi.recentSender, Msg.BArrayToObjects(barray));
            else
                throw new IsisException("Redeliver");
        }

        private void cbAction(Msg m, int vid, int msgid, Address sender, object[] obs)
        {
            if (obs.Length == 0)
                return;
            while (obs.Length == 1 && obs[0].GetType().Equals(typeof(Msg)))
            {
                // Special for sending INITIALVIEW via OOB, which show up in an unusual format because the receiver isn't yet
                // a member of any groups, forcing Isis to use a form of pure p2p send that double-encapsulates the true message
                // This "hack" makes it possible for g.Reply() to work as if it was within a group...
                m = (Msg)obs[0];
                obs = Msg.BArrayToObjects(m.payload);
            }
            try
            {
                SetCurMsg(m);
                obs = fixObs(obs);
                if (hasUniversalHandlers && isP2PThread)
                {
                    if (rcode(obs) >= 0)
                        using (new LockAndElevate(UniversalP2PHandlers.uhListLock))
                            foreach (UHCallBack uc in UniversalP2PHandlers.uhList)
                                uc.uhProc(obs);
                }
                else if (hasUniversalHandlers)
                {
                    if (rcode(obs) >= 0)
                        using (new LockAndElevate(UniversalMHandlers.uhListLock))
                            foreach (UHCallBack uc in UniversalMHandlers.uhList)
                                uc.uhProc(obs);
                }
                List<CallBack> cblist;
                int request = rcode(obs), cbCnt = 0;
                if (request == Isis.CRYPTOWRAPPED || request == Isis.CLIENTWRAPPED)
                    using (new LockAndElevate(rdiLock))
                        rdiList.Add(new redeliveryInfo(vid, msgid, sender, Thread.CurrentThread));
                try
                {
                    cblist = Handlers[request].hList;
                }
                catch
                {
                    cblist = null;
                }
                if (cblist == null)
                {
                    if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                        Isis.WriteLine("WARNING: no registered callbacks known for request " + Isis.rToString((int)obs[0]) + ", msg " + vid + ":" + msgid + " from " + sender);
                    using (new LockAndElevate(IsisSystem.RTS.Lock))
                        IsisSystem.RTS.Discarded++;
                    return;
                }
                object[] args = new object[obs.Length - 1];
                for (int i = 0; i < args.Length; i++)
                    args[i] = obs[i + 1];
                if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.CALLBACKS)) != 0)
                    Isis.WriteLine("In group " + gname + " about to do callbacks for request " + Isis.rToString((int)obs[0]) + ", msg " + vid + ":" + msgid + " from " + sender);
                List<CallBack> toDo = new List<CallBack>();
                foreach (CallBack cb in cblist)
                    if (TypeMatch(obs, cb))
                    {
                        ++cbCnt;
                        toDo.Add(cb);
                    }

                foreach (CallBack cb in toDo)
                {
                    if ((IsisSystem.Debug & IsisSystem.CALLBACKS) != 0)
                        ReportCb(cb.cbProc, args);
                    long before = Isis.NOW();
                    if (cb.withLock)
                    {
                        new Thread(delegate()
                        {
                            try
                            {
                                Thread.CurrentThread.Name = "Callback with lock";
                                using (new ILock(ILock.LLENTRY, gaddr))
                                    cb.cbProc.doUpcall(args);
                            }
                            catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                            IsisSystem.ThreadTerminationMagic();
                        }).Start();
                    }
                    else
                        cb.cbProc.doUpcall(args);
                    if ((IsisSystem.Debug & IsisSystem.DELAYS) != 0 && (Isis.NOW() - before) > 500)
                        Isis.WriteLine("WARNING: Callback to request " + Isis.rToString((int)obs[0]) + ", msg " + vid + ":" + msgid + " from " + sender + " took " + (Isis.NOW() - before) + "ms");
                }
                if (cbCnt == 0)
                {
                    string ts = " ";
                    for (int i = 1; i < obs.Length; i++)
                        ts += obs[i].GetType() + " ";
                    Isis.WriteLine("WARNING: No matching type signature for incoming msg " + vid + ":" + msgid + " from " + sender + "; [<" + gname + ">." + Isis.rToString((int)obs[0]) + "(" + ts + ")");
                    return;
                }
                Msg replyTo = getReplyTo();
                if (replyTo != null)
                {
                    if ((IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0)
                        Isis.WriteLine("WARNING: Sending an automatic NULLREPLY " + (Thread.CurrentThread.Name == null ? " (unnamed thread" : Thread.CurrentThread.Name));
                    doNullReply();
                }
                if ((IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0)
                    Isis.WriteLine("In group " + gname + " after callbacks for request " + Isis.rToString((int)obs[0]) + ", msg " + vid + ":" + msgid + " from " + sender);
            }
            finally
            {
                SetCurMsg(null);
            }
        }

        internal const int RT_REPLY = 1;
        internal const int RT_NULL = 2;
        internal const int RT_ABORT = 3;

        /// <summary>
        /// Invoked by a recipient of a Query to send this member's contribution towards the result of the Query
        /// </summary>
        /// <param name="obs">variable-length argument list of values to pass back to the caller</param>
        public void Reply(params object[] obs)
        {
            if (!IsisCallStart())
                return;
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            doReply(obs);
            IsisCallDone();
        }

        internal static LockObject Rlock = new LockObject("Rlock");

        internal void doReply(params object[] obs)
        {
            if (GroupOpen == false && WasOpen)
                return;
            Msg replyTo;
            using (new LockAndElevate(Rlock))
                replyTo = getReplyToAndClear();
            if (replyTo == null)
                throw new IsisException("Attempted to reply twice to same message, or to a message that wasn't a query");
            if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                Isis.WriteLine("Sending reply to " + replyTo.sender + ", " + replyTo.vid + ":" + replyTo.msgid);
            Type t;
            if (obs.Length > 16 && Msg.AllHaveSameType(obs, out t))
                obs = new object[] { obs };
            byte[] result = Msg.toBArray(obs);
            bool enciphered = false;
            if ((flags & G_SECURE) != 0 && (replyTo.flags & Msg.ENCIPHEREDREPLY) != 0)
            {
                enciphered = true;
                result = cipherBuf(result);
            }
            bool deliverToOracle = (replyTo.flags & Msg.SENTBYORACLE) != 0;
            byte[] buffer = Msg.toBArray(RT_REPLY, replyTo.vid, replyTo.msgid, deliverToOracle, enciphered, result);
            Isis.PendingLeaderOps plos;
            using (new LockAndElevate(groupLock))
                plos = NotifyDALOnReply;
            if (plos != null && plos.reqMsg == replyTo)
                Isis.DALReplyNotify(this, new Msg(gaddr, Msg.ISREPLY, replyTo.sender, Msg.NewMsgAsBArray(Isis.my_address, gaddr, theView.viewid, -1, 0L, 0, 0, 0, buffer), theView.viewid, -1), plos, replyTo);
            ReliableSender.SendP2P(Msg.ISREPLY, replyTo.sender, rgroup(replyTo.gaddr), theView == null ? 0 : theView.viewid, ReliableSender.P2PSequencer.NextP2PSeqn("reply/1", replyTo.sender),
                buffer, true, null, replyTo);
        }

        // This is a little trick to support the Client of a group logic.  If the reply message shows that the request was sent in ISISMEMBERS, make sure
        // the reply shows up listing ISISMEMBERS as the group so that the reply will match the reply-wait on the client side.
        internal Group rgroup(Address gaddr)
        {
            if (gaddr != null && gaddr.Equals(Isis.ISISMEMBERS.gaddr))
                return Isis.ISISMEMBERS;
            return this;
        }

        /// <summary>
        /// Invoked by the recipient of a Query if this member will not participate in performing the Query
        /// </summary>
        public void NullReply()
        {
            if (!IsisCallStart())
                return;
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            doNullReply();
            IsisCallDone();
        }

        internal void doNullReply()
        {
            Msg replyTo = getReplyToAndClear();
            if (GroupOpen == false || replyTo == null || (replyTo.flags & Msg.NEEDSREPLY) == 0)
                throw new IsisException("NullReply: you already replied to this message, or it wasn't a query");
            if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                Isis.WriteLine("NullReply to " + replyTo.sender + " msgid " + replyTo.vid + ":" + replyTo.msgid);
            bool deliverToOracle = (replyTo.flags & Msg.SENTBYORACLE) != 0;
            ReliableSender.SendP2P(Msg.ISREPLY, replyTo.sender, rgroup(replyTo.gaddr), theView == null ? 0 : theView.viewid, ReliableSender.P2PSequencer.NextP2PSeqn("reply/2", replyTo.sender),
                Msg.toBArray(RT_NULL, replyTo.vid, replyTo.msgid, deliverToOracle, (flags & G_SECURE) != 0), true, null, replyTo);
        }

        /// <summary>
        /// Invoked by the recipient of a Query to force an IsisAbort exception in the caller.
        /// </summary>
        /// <param name="reason">string giving the reason for aborting this query</param>
        public void AbortReply(string reason)
        {
            if (!IsisCallStart())
                return;
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            doAbortReply(reason);
            IsisCallDone();
        }

        internal void doAbortReply(string reason)
        {
            Msg replyTo = getReplyToAndClear();
            if (GroupOpen == false || replyTo == null || (replyTo.flags & Msg.NEEDSREPLY) == 0)
                throw new IsisException("AbortReply: you already replied to this message, or it wasn't a query");
            if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                Isis.WriteLine("AbortReply to " + replyTo.vid + ":" + replyTo.msgid);
            bool deliverToOracle = (replyTo.flags & Msg.SENTBYORACLE) != 0;
            ReliableSender.SendP2P(Msg.ISREPLY, replyTo.sender, rgroup(replyTo.gaddr), theView == null ? 0 : theView.viewid, ReliableSender.P2PSequencer.NextP2PSeqn("reply/3", replyTo.sender),
                Msg.toBArray(RT_ABORT, replyTo.vid, replyTo.msgid, deliverToOracle, (flags & G_SECURE) != 0, reason), true, null, replyTo);
        }

        internal static List<Address> RecentlyLeft = new List<Address>();
        internal LockObject RecentlyLeftLock = new LockObject("RecentlyLeftLock");

        /// <summary>
        /// Invoked by the caller to leave a particular group
        /// </summary>
        public void Leave()
        {
            if (!IsisCallStart())
                return;
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            using (new LockAndElevate(RecentlyLeftLock))
                RecentlyLeft.Add(gaddr);
            Isis.OnTimer(5 * 60 * 1000, delegate() { using (new LockAndElevate(RecentlyLeftLock)) RecentlyLeft.Remove(gaddr); });
            if (HasFirstView == false)
                GroupClose();
            doLeave(0, new Group[] { this });
            IsisCallDone();
        }

        /// <summary>
        /// Invoked by the caller to leave a set of groups in one atomic action
        /// </summary>
        /// <param name="groups">list of groups to leave</param>
        public static void multiLeave(Group[] groups)
        {
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            foreach (Group g in groups)
                if (!g.HasFirstView)
                    throw new IsisException("multiLeave: can only be called by a current member of <" + g.gname + ">");
            doLeave(0, groups);
        }

        /// <summary>
        /// Invoked by the master process to remove a set of workers from a set of groups in one atomic action
        /// </summary>
        /// <param name="workers">list of processes to remove from the list of groups</param>
        /// <param name="groups">list of groups</param>
        public static void multiLeave(Address[] workers, Group[] groups)
        {
            if (IsisSystem.IsisActive == false)
                throw new IsisShutdown("Isis inactive");
            doLeave(0, workers, groups);
        }

        internal static void doLeave(int level, Address[] oldMembers, Group[] groups)
        {
            doLeave(level, virtualGroup(oldMembers), groups);
        }

        internal static void doLeave(int level, Group[] groups)
        {
            doLeave(level, Isis.my_address, groups);
        }

        internal static void doLeave(int level, Address who, Group[] groups)
        {
            Address[] gaddrs = new Address[groups.Length];
            String[] gnames = new String[groups.Length];
            int[] flags = new int[groups.Length];
            int cnt = 0;
            foreach (Group g in groups)
            {
                g.Flush();
                flags[cnt] = g.flags;
                gnames[cnt] = g.gname;
                gaddrs[cnt++] = g.gaddr;
            }

            if (Isis.ClientOf == null)
                Isis.ORACLE.QueryToBA(ALL, new Timeout(Isis.ISIS_DEFAULTTIMEOUT, Timeout.TO_FAILURE, "LEAVE"), Isis.LEAVE, who, 0, gnames, gaddrs, flags, ++IsisSystem.IsisJoinCounter);
            else while (Isis.ClientOf != null)
                    if (Isis.ORACLE.doP2PQuery(Isis.ClientOf, new Timeout(Isis.ISIS_DEFAULTTIMEOUT, Timeout.TO_FAILURE, "RELAYLEAVE"), Isis.RELAYLEAVE, who, Isis.ORACLE.uids++, gnames, gaddrs, flags).Length != 0)
                        break;

            TerminationWait(groups);
        }

        private static void TerminationWait(Group[] groups)
        {
            foreach (Group g in groups)
                while (IsisSystem.IsisActive && g.GroupOpen)
                {
                    IsisSystem.RTS.ThreadCntrs[13]++;
                    Thread.Sleep(250);
                }
        }

        internal Address[] getLiveMembers()
        {
            if ((flags & G_ISLARGE) != 0)
                return new Address[] { theView.members[0] };
            return getLiveMembers(theView);
        }

        internal static Address[] getLiveMembers(View v)
        {
            if (v == null) return new Address[0];
            Address[] LiveGuys = new Address[v.nLive()];
            int i = 0, j = 0;
            while (i < v.members.Length)
            {
                if (v.hasFailed[i] == false)
                    LiveGuys[j++] = v.members[i];
                ++i;
            }
            return LiveGuys;
        }


        internal class gStashNode
        {
            internal Address sender;
            internal Address gaddr;
            internal int minStable;
            internal Msg m;
            internal int deliveryAttempts;

            internal gStashNode(Address s, Address ga, int ms, Msg msg)
            {
                sender = s; gaddr = ga; minStable = ms; m = msg;
            }
        }

        internal static List<gStashNode> stash = new List<gStashNode>();
        internal static LockObject stashLock = new LockObject("stashLock");
        internal volatile static bool stashNonEmpty = false;
        internal LockObject P2PStashLock = new LockObject("P2PStashLock");
        internal List<Msg> P2PStash = new List<Msg>();

        internal static void stashMsg(Address sender, Address gaddr, int minStable, Msg m)
        {
            using (new LockAndElevate(stashLock))
            {
                stash.Add(new gStashNode(sender, gaddr, minStable, m));
                stashNonEmpty = true;
            }
        }

        internal static void stashCheck()
        {
            bool nullGroup = false;
            List<Group> replayList = new List<Group>();
            using (new LockAndElevate(stashLock))
                foreach (gStashNode gsn in stash)
                {
                    Group g = Group.doLookup(gsn.gaddr);
                    if (g != null && !replayList.Contains(g))
                        replayList.Add(g);
                    else if (gsn.gaddr.isNull())
                        nullGroup = true;
                }
            if (nullGroup)
                replayStash(null);
            foreach (Group g in replayList)
            {
                replayStash(g);
                g.replayP2PStash();
            }
        }

        internal void replayP2PStash()
        {
            using (new LockAndElevate(P2PStashLock))
            {
                List<Msg> newList = new List<Msg>();
                foreach (Msg m in P2PStash)
                    if (Msg.checkView(m))
                        incomingP2P.put(m);
                    else
                        newList.Add(m);
                P2PStash = newList;
            }
        }

        internal static void replayStash(Group g)
        {
            int ms = -1;
            List<gStashNode> cblist = new List<gStashNode>();
            List<gStashNode> nstash = new List<gStashNode>();
            using (new LockAndElevate(stashLock))
            {
                foreach (gStashNode gsn in stash)
                    if ((g == null) ? gsn.gaddr.isNull() : g.gaddr.Equals(gsn.gaddr))
                    {
                        ms = Math.Max(ms, gsn.minStable);
                        cblist.Add(gsn);
                    }
                    else
                        nstash.Add(gsn);
                stash = nstash;
                stashNonEmpty = (stash.Count() > 0);
            }
            foreach (gStashNode gsn in cblist)
                if (ReliableSender.doGotIncoming(Msg.ISGRPP2P, gsn.gaddr, gsn.sender, ms, gsn.m, g, false) == false)
                    using (new LockAndElevate(stashLock))
                        if (++gsn.deliveryAttempts > 10)
                            Isis.WriteLine("WARNING: giving up on delivery for stashed message: " + gsn.m);
                        else
                        {
                            stash.Add(gsn);
                            stashNonEmpty = true;
                        }
            if ((g.flags & G_ISLARGE) != 0)
            {
                List<ReliableSender.MsgDesc> toReplay = new List<ReliableSender.MsgDesc>();
                using (new LockAndElevate(g.GotAMsgLock))
                {
                    using (new LockAndElevate(ReliableSender.PendingSendBufferLock))
                        foreach (ReliableSender.MsgDesc lgmd in ReliableSender.LgPendingSendBuffer)
                            if (lgmd.group == g && (lgmd.MsgId == -1 || lgmd.MsgId >= g.nextMsgid))
                                toReplay.Add(lgmd);
                }
                foreach (ReliableSender.MsgDesc lgmd in toReplay)
                    ReliableSender.RecvBB.put(lgmd.theBytes);
            }
        }

        internal static string dumpStash()
        {
            if (stash.Count() == 0)
                return "";
            string s = "LIST OF EARLY-RECEIVED GRP P2P MSGS:\r\n";
            using (new LockAndElevate(stashLock))
                foreach (gStashNode gsn in stash)
                    s += "  STASHED: " + gsn.m + "\r\n";
            return s;
        }

        /// <summary>
        /// Called to register a new Aggregator method for this group. 
        /// </summary>
        /// <param name="hisAggregator">a method to perform aggregation computations, must implement the IAggregator interface</param>
        /// <typeparam name="KeyType">Type for keys (must be registered first via Msg.registerType, must implement IEqualityComparer)</typeparam>
        /// <typeparam name="ValueType">Type for values (must be registered first via Msg.registerType)</typeparam>
        /// <remarks> 
        /// Called to register a new Aggregator method for this group.  It must implement the IAggregator interface, which can
        /// also be accessed as an in-line delegate type of type AggregatorDel
        /// </remarks>
        public void RegisterAggregator<KeyType, ValueType>(Aggregator<KeyType, ValueType> hisAggregator)
        {
            RegisterAggregator<KeyType, ValueType>(hisAggregator, new Timeout(Isis.ISIS_DEFAULTTIMEOUT, Timeout.TO_AGGFAILURE, "AGGFAILURE"));
        }

        /// <summary>
        /// Called to register a new Aggregator method for this group. 
        /// </summary>
        /// <param name="hisAggregator">a method to perform aggregation computations, must implement the IAggregator interface</param>
        /// <param name="AggTimeout">A timeout indicating the maximum delay before the aggregation fails and the action this triggers</param>
        /// <typeparam name="KeyType">Type for keys (must be registered first via Msg.registerType, must implement IEqualityComparer)</typeparam>
        /// <typeparam name="ValueType">Type for values (must be registered first via Msg.registerType)</typeparam>
        /// <remarks> 
        /// Called to register a new Aggregator method for this group.  It must implement the IAggregator interface, which can
        /// also be accessed as an in-line delegate type of type AggregatorDel
        /// </remarks>
        public void RegisterAggregator<KeyType, ValueType>(Aggregator<KeyType, ValueType> hisAggregator, Timeout AggTimeout)
        {
            if (AggTimeout.action != Timeout.TO_AGGFAILURE)
                throw new ArgumentException("registerAggregation: Illegal failure action");
            doRegisterAggregator<KeyType, ValueType>(hisAggregator, AggTimeout);
            flags |= G_HASUAGG;
        }

        // Internal one is identical except it doesn't set the user-defined-aggregators flag, which affects token rate
        internal void doRegisterAggregator<KeyType, ValueType>(Aggregator<KeyType, ValueType> hisAggregator)
        {
            doRegisterAggregator<KeyType, ValueType>(hisAggregator, new Timeout(Isis.ISIS_DEFAULTTIMEOUT * 2, Timeout.TO_AGGFAILURE, "REGISTER.AGGREGATOR"));
        }

        // Internal one is identical except it doesn't set the user-defined-aggregators flag, which affects token rate
        internal void doRegisterAggregator<KeyType, ValueType>(Aggregator<KeyType, ValueType> hisAggregator, Timeout AggTimeout)
        {
            Type KT = typeof(KeyType), VT = typeof(ValueType);
            if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                Isis.WriteLine("registerAggregator<" + KT + "><" + VT + "> " + hisAggregator);
            if (HasFirstView)
                throw new IsisException("Illegal to register aggregator after doing a group join");
            if (isTrackingProxy)
                throw new IsisException("registerAggregator called in a tracking proxy");
            Msg.ConfirmRegistered(KT);
            Msg.ConfirmRegistered(VT);
            foreach (AggInfo ag in AggTypes)
                if (ag.KT.Equals(typeof(KeyType)) && ag.VT.Equals(typeof(ValueType)))
                    throw new IsisException("Group.registerAggregator: Type signature <" + typeof(KeyType) + "," + typeof(ValueType) + "> already registered in Group " + gname
                        + " (hint: disambiguate aggregator instances using a structured key instead of trying to re-register the type)");
            ConstructorInfo ci = typeof(Aggregation<KeyType, ValueType>).GetConstructor(new Type[] { typeof(Group), typeof(int), typeof(int), typeof(Aggregator<KeyType, ValueType>), typeof(Timeout) });
            if (ci == null)
                throw new IsisException("Aggregator " + GetType() + " lacks a constructor(Group g, Aggregator<KeyType, ValueType> del)");
            if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                Isis.WriteLine("Group <" + gname + ">... registerAggregator for type " + KT + ":" + VT);
            AggTypes.Add(new AggInfo(KT, VT, ci, this, hisAggregator, AggTimeout));
        }

        internal string AggState()
        {
            string s = "";
            using (new LockAndElevate(AggListLock))
                if (AggList != null)
                    for (int l = 0; l < AggList.Length; l++)
                        foreach (IAggregateEventHandler ae in AggList[l])
                        {
                            string astate = ae.AggState();
                            if (astate.Length > 0)
                                s += "            Aggregator " + ae.AggName() + " level " + l + "\r\n" + astate;
                        }
            if (s.Length > 0)
                return "         AGGREGATION STATE:\r\n" + s;
            return s;
        }

        internal void InterruptAggregationWaits()
        {
            if ((flags & G_ISLARGE) != 0)
            {
                tokenInfo theToken;
                using (new LockAndElevate(TokenLock))
                    theToken = this.theToken;
                if (theToken == null)
                    return;
                for (int level = 0; level < theToken.nlevels; level++)
                    foreach (IAggregateEventHandler ae in theToken.theGroup.AggList[level])
                        try
                        {
                            ae.AggEvent(BreakWaits);
                        }
                        catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
            }
            else using (new LockAndElevate(AggListLock))
                    if (AggList != null)
                        for (int level = 0; level < AggList.Length; level++)
                            foreach (IAggregateEventHandler ae in AggList[level])
                                try
                                {
                                    ae.AggEvent(BreakWaits);
                                }
                                catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
        }

        /// <summary>
        /// Called by the user to set a local value for an aggregator.
        /// </summary>
        /// <typeparam name="KeyType">Type for keys (must be registered first via Msg.registerType)</typeparam>
        /// <typeparam name="ValueType">Type for values (must be registered first via Msg.registerType)</typeparam>
        /// <param name="key">the key identifying this aggregation</param>
        /// <param name="val">the value this process contributes towards the aggregation</param>
        /// <remarks>
        /// Called by the user to set a local value for an aggregator.  The key identifies the aggregation instance and the value is the local value.  Called just once for any
        /// particular key value.  If you need to have multiple episodes of aggregation for a single key type, encode the episode id into a keytype class.
        /// </remarks>
        public void SetAggregatorValue<KeyType, ValueType>(KeyType key, ValueType val)
        {
            if (!HasFirstView)
                return;
            if (myLoggingFcn != null)
                myLoggingFcn(IL_AGGDVALUE, IL_START, Isis.my_address, newLoggingId(), key, val);
            SetAggregatorValueAndVID<KeyType, ValueType>(theView.viewid, key, val);
        }

        // Internal version used ONLY in the flush algorithm
        internal void SetAggregatorValueAndVID<KeyType, ValueType>(int vid, KeyType key, ValueType val)
        {
            tokenInfo theToken = null;
            if ((flags & G_ISLARGE) != 0)
                using (new LockAndElevate(TokenLock))
                    theToken = this.theToken;
            if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                Isis.WriteLine("SetAggregatorValueAndVID in <" + gname + "> for " + typeof(KeyType) + ":" + typeof(ValueType) + ", key=" + key + ", value=" + val + " and with VID=" + vid);
            int idx = 0;
            interesting = true;
            foreach (AggInfo ai in AggTypes)
            {
                if (ai.KT.Equals(typeof(KeyType)) && ai.VT.Equals(typeof(ValueType)))
                {
                    if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                        Isis.WriteLine("SetAggregatorValueAndVID: idx " + idx + " matched ai.KT<" + ai.KT + "> == KeyType<" + typeof(KeyType) + "> and ai.VT<" + ai.VT + "> == ValueType<" + typeof(ValueType) + ">");
                    if ((flags & G_ISLARGE) != 0)
                        ((Aggregation<KeyType, ValueType>)theToken.theGroup.AggList[0].ElementAt(idx)).Set(vid, key, val);
                    else
                    {
                        using (new LockAndElevate(AggListLock))
                            ((Aggregation<KeyType, ValueType>)AggList[0].ElementAt(idx)).Set(vid, key, val);
                    }
                    return;
                }
                else if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("SetAggregatorValueAndVID: ai.KT<" + ai.KT + "> != KeyType<" + typeof(KeyType) + "> or ai.VT<" + ai.VT + "> != ValueType<" + typeof(ValueType) + ">");
                ++idx;
            }
            throw new IsisException("SetAggregatorValue: No registered aggregator for group " + gname + " has type signature  <" + typeof(KeyType) + "," + typeof(ValueType) + ">");
        }

        /// <summary>
        /// Called by the group leader to obtain the result from an aggregation operation identified by a key.
        /// </summary>
        /// <typeparam name="KeyType">Type for keys in the aggregator</typeparam>
        /// <typeparam name="ValueType">Type for values in the aggregator</typeparam>
        /// <param name="key">A key identifying the aggregation of interest</param>
        /// <returns>The result of the aggregation</returns>
        /// <remarks>
        /// Called by the user to obtain the result from an aggregation operation identified by a key, but can only be performed in the large group leader, which is the zero-ranked member of the view.
        /// The leader should multicast the results to the members if you wish the members to also have this information.  Throws an exception if called in a non-leader.  A timeout will be posted 
        /// (see RegisterAggregator) and in the event that it triggers before the aggregation succeeds, this breaks the wait and returns an aggregation failure exception to the caller.
        /// </remarks>
        public ValueType GetAggregatorResult<KeyType, ValueType>(KeyType key)
        {
            if (!IsisCallStart())
                return default(ValueType);
            tokenInfo theToken = null;
            if ((flags & G_ISLARGE) != 0)
                using (new LockAndElevate(TokenLock))
                    theToken = this.theToken;
            long mylid = 0;
            if (myLoggingFcn != null)
                myLoggingFcn(IL_AGGWAIT, IL_START, Isis.my_address, mylid = newLoggingId(), key);
            try
            {
                if ((flags & G_ISLARGE) != 0 && (theToken == null || !theToken.IAmLgOwner))
                    throw new IsisException("GetAggregatorResult: Can only be called by the rank-zero member");
                int idx = 0;
                foreach (AggInfo ai in AggTypes)
                {
                    if (ai.KT.Equals(typeof(KeyType)) && ai.VT.Equals(typeof(ValueType)))
                    {
                        ValueType val;
                        if ((flags & G_ISLARGE) != 0)
                        {
                            val = ((Aggregation<KeyType, ValueType>)theToken.theGroup.AggList[theToken.nlevels - 1].ElementAt(idx)).Get(key);
                        }
                        else
                        {
                            Aggregation<KeyType, ValueType> ag;
                            using (new LockAndElevate(AggListLock))
                                ag = (Aggregation<KeyType, ValueType>)AggList[AggList.Length - 1].ElementAt(idx);
                            val = ag.Get(key);
                        }
                        if (val == null)
                            throw new AggregationFailed(2);
                        if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                            Isis.WriteLine("GetAggregatorResult in <" + gname + "> for " + typeof(KeyType) + ":" + typeof(ValueType) + ", key=" + key + ", value=" + val);
                        if (myLoggingFcn != null)
                            myLoggingFcn(IL_AGGWAIT, IL_DONE, Isis.my_address, mylid, key, val);
                        IsisCallDone();
                        return val;
                    }
                    ++idx;
                }
                throw new IsisException("GetAggregatorResult: No registered aggregator for group " + gname + " has type signature  <" + typeof(KeyType) + "," + typeof(ValueType) + ">");
            }
            catch (IsisShutdown)
            {
                throw new AggregationFailed(3);
            }
        }

        internal const int GotRValue = 0;
        internal const int GotDValue = 1;
        internal const int GotAggRes = 2;
        internal const int GetAggState = 3;
        internal const int GetRank0State = 4;
        internal const int BreakWaits = 5;

        internal const int LEN = 0;
        internal const int VIDS = 1;
        internal const int KEYS = 2;
        internal const int VALS = 3;

        internal class binfo
        {
            internal Type KT, VT;
            internal object theBarrier = new object();
            internal static List<KeyValuePair<object, object>> theResults = new List<KeyValuePair<object, object>>();
            internal static LockObject theBarrierLock = new LockObject("theBarrierLock");
            internal static List<binfo> theBarrierList = new List<binfo>();

            internal binfo(Type kt, Type vt)
            {
                KT = kt;
                VT = vt;
            }

            // Called while holding a Monitor lock on the AggResult object
            internal object getResult(object key)
            {
                if (!key.GetType().Equals(KT))
                    throw new IsisException("binfo: getResult with wrong key type");
                foreach (var res in theResults)
                    if (res.Key.Equals(key))
                    {
                        theResults.Remove(res);
                        return res.Value;
                    }
                return null;
            }


            // Called while holding a Monitor lock on the AggResult object
            internal void saveResult(object key, object value)
            {
                if (!key.GetType().Equals(KT) || !value.GetType().Equals(VT))
                    throw new IsisException("binfo: saveResult with wrong key or value type");
                foreach (var res in theResults)
                    if (res.Key.Equals(key))
                    {
                        theResults.Remove(res);
                        break;
                    }
                theResults.Add(new KeyValuePair<object, object>(key, value));
            }

            internal static binfo getBarrierObject(Type kt, Type vt)
            {
                using (new LockAndElevate(theBarrierLock))
                {
                    foreach (var bi in theBarrierList)
                        if (bi.KT.Equals(kt) && bi.VT.Equals(vt))
                            return bi;
                    binfo newBi = new binfo(kt, vt);
                    theBarrierList.Add(newBi);
                    return newBi;
                }
            }

            internal static void resetBarrierList()
            {
                using (new LockAndElevate(theBarrierLock))
                    theBarrierList = new List<binfo>();
            }
        }

        /// <exclude>
        /// <summary>
        /// Internal class used by Isis to perform Aggregations, public only to comply with C# scope rules. 
        /// </summary>
        /// <typeparam name="KeyType">Type for keys</typeparam>
        /// <typeparam name="ValueType">Type for values</typeparam>
        /// </exclude>
        public class Aggregation<KeyType, ValueType> : IAggregateEventHandler
        {
            internal LockObject Alock = new LockObject("Alock");    // Used to protect shared variables
            internal binfo AggResult = binfo.getBarrierObject(typeof(KeyType), typeof(ValueType));
            internal Group myGroup;
            internal int myLevel;
            internal int myVid;
            Aggregator<KeyType, ValueType> theAggMethod;
            volatile internal bool registered = false;
            internal Timeout AggTimeout;

            internal Address[] Participants(KeyType key, View v)
            {
                if (!typeof(KeyType).IsGenericType || typeof(KeyType).GetGenericTypeDefinition() != typeof(QueryKey<>))
                    return v.members;
                else
                {
                    Address[] dests = ((QKD)key).GetDestsToArray(myGroup, myVid);
                    if (dests != null)
                        return dests;
                    // Occurs if the view we are asking about has become stale
                    throw new AggregationFailed(4);
                }
            }

            /// <exclude></exclude>
            public Aggregation(Group g, int vid, int level, Aggregator<KeyType, ValueType> del, Timeout TO)
            {
                myGroup = g;
                myVid = vid;
                myLevel = level;
                theAggMethod = del;
                AggTimeout = TO;
                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("Aggregation constructor in <" + g.gname + "> for vid=" + vid + ": creating a new object for Aggregator " + del);
            }

            /// <exclude></exclude>
            public Type GetKeyType()
            {
                return typeof(KeyType);
            }

            /// <exclude></exclude>
            public Type GetValueType()
            {
                return typeof(ValueType);
            }

            /// <exclude>
            /// <summary>
            /// Internal method used by Isis as part of the Aggregation layer, public only to comply with C# scope rules.
            /// </summary>
            /// <returns>dictionary of ldTriples coerced to object type</returns>
            /// </exclude>
            public object GetDValues()
            {
                return (object)ldValues;
            }

            /// <exclude>
            /// <summary>
            /// Internal method used by Isis as part of the Aggregation layer, public only to comply with C# scope rules.
            /// </summary>
            /// <param name="fromBelow">dictionary of ldTriples from below</param>
            /// </exclude>
            public void LoadDValues(object fromBelow)
            {
                Dictionary<int, ldTuple<KeyType, ValueType>> belowldValues = (Dictionary<int, ldTuple<KeyType, ValueType>>)fromBelow;
                List<KeyValuePair<int, ldTuple<KeyType, ValueType>>> needDValue = new List<KeyValuePair<int, ldTuple<KeyType, ValueType>>>();
                using (new LockAndElevate(Alock))
                    foreach (KeyValuePair<int, ldTuple<KeyType, ValueType>> kvp in belowldValues)
                        if (kvp.Value.hasDValue)
                            needDValue.Add(kvp);
                foreach (KeyValuePair<int, ldTuple<KeyType, ValueType>> kvp in needDValue)
                    SetDValue(kvp.Value.vid, kvp.Value.key, kvp.Value.dValue);
            }

            /// <exclude>
            /// <summary>
            /// Internal method used by Isis as part of the Aggregation layer, public only to comply with C# scope rules.
            /// </summary>
            /// <param name="etype">event type</param>
            /// <returns>byte[] in the case of GetAggState, else null</returns>
            /// </exclude>
            public byte[] AggEvent(int etype)
            {
                return AggEvent(etype, 0, null, null, 0);
            }

            /// <exclude>
            /// <summary>
            /// Internal method used by Isis as part of the Aggregation layer, public only to comply with C# scope rules.
            /// </summary>
            /// <param name="etype">event type</param>
            /// <param name="key">key value</param>
            /// <param name="arg">associated argument, or null</param>
            /// <param name="offset">index into key/values array</param>
            /// <param name="vid">viewid if successful, -viewid if failed</param>
            /// <returns>byte[] in the case of GetAggState, else null</returns>
            /// </exclude>
            public byte[] AggEvent(int etype, int vid, object key, object arg, int offset)
            {
                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("Aggregator in <" + myGroup.gname + ">: AggEvent(type=" + etype + ")");
                switch (etype)
                {
                    case GotRValue:
                        KeyType[] KTA = (KeyType[])key;
                        ValueType[] VTA = (ValueType[])arg;
                        if (KTA != null && KTA.Length > offset && VTA != null && VTA.Length > offset)
                        {
                            SetRValue(vid, KTA[offset], VTA[offset]);
                            if (myGroup.myLoggingFcn != null)
                                myGroup.myLoggingFcn(IL_AGGLVALUE, IL_START, Isis.my_address, -1, KTA[offset], VTA[offset]);
                        }
                        break;

                    case GotDValue:
                        KTA = (KeyType[])key;
                        VTA = (ValueType[])arg;
                        if (KTA != null && KTA.Length > offset && VTA != null && VTA.Length > offset)
                        {
                            SetDValue(vid, KTA[offset], VTA[offset]);
                            if (myGroup.myLoggingFcn != null)
                                myGroup.myLoggingFcn(IL_AGGDVALUE, IL_START, Isis.my_address, -1, KTA[offset], VTA[offset]);
                        }
                        break;

                    case GotAggRes:
                        KTA = (KeyType[])key;
                        VTA = (ValueType[])arg;
                        if (KTA != null && KTA.Length > offset && VTA != null && VTA.Length > offset)
                            SetAggResult(vid, KTA[offset], VTA[offset]);
                        break;

                    case GetRank0State:
                        // Send known dvalues, and garbage collect them
                        int idx = 0;
                        List<KeyValuePair<int, ldTuple<KeyType, ValueType>>> toSend = new List<KeyValuePair<int, ldTuple<KeyType, ValueType>>>();
                        using (new LockAndElevate(Alock))
                        {
                            Dictionary<int, ldTuple<KeyType, ValueType>> nldValues = new Dictionary<int, ldTuple<KeyType, ValueType>>(Group.tokenInfo.RINGSIZE * 2);

                            foreach (KeyValuePair<int, ldTuple<KeyType, ValueType>> kvp in ldValues)
                                if (kvp.Value.hasDValue)
                                    toSend.Add(new KeyValuePair<int, ldTuple<KeyType, ValueType>>(kvp.Key.GetHashCode(), kvp.Value));
                                else
                                    nldValues.Add(kvp.Key.GetHashCode(), kvp.Value);
                            ldValues = nldValues;
                        }

                        int len = toSend.Count();
                        int[] vids = new int[len];
                        KeyType[] keys = new KeyType[len];
                        ValueType[] values = new ValueType[len];

                        foreach (AggInfo ai in myGroup.AggTypes)
                            if (ai.KT.Equals(typeof(KeyType)) && ai.VT.Equals(typeof(ValueType)))
                            {
                                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                                    Isis.WriteLine("Aggregator: GetRank0State<" + typeof(KeyType) + ":" + typeof(ValueType) + ">, computed a vector of length " + toSend.Count() + " containing: ");
                                foreach (KeyValuePair<int, ldTuple<KeyType, ValueType>> kvp in toSend)
                                {
                                    keys[idx] = kvp.Value.key;
                                    vids[idx] = kvp.Value.vid;
                                    values[idx] = kvp.Value.dValue;
                                    if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                                        Isis.WriteLine("       { vid = " + vids[idx] + ", key=" + keys[idx] + ", value=" + values[idx] + " }");
                                    ++idx;
                                }
                                return (Msg.toBArray(idx, vids, keys, values));
                            }
                        break;

                    case GetAggState:
                        // Send values for which we know both the lvalue and the dvalue, and garbage collect them
                        idx = 0;
                        toSend = new List<KeyValuePair<int, ldTuple<KeyType, ValueType>>>();
                        using (new LockAndElevate(Alock))
                        {
                            Dictionary<int, ldTuple<KeyType, ValueType>> nldValues = new Dictionary<int, ldTuple<KeyType, ValueType>>(Group.tokenInfo.RINGSIZE * 2);

                            foreach (KeyValuePair<int, ldTuple<KeyType, ValueType>> kvp in ldValues)
                                if (kvp.Value.hasDValue && kvp.Value.hasRValue)
                                    toSend.Add(new KeyValuePair<int, ldTuple<KeyType, ValueType>>(kvp.Key.GetHashCode(), kvp.Value));
                                else if (kvp.Value.hasDValue && Isis.NOW() - kvp.Value.DValueTime > AggTimeout.when)
                                    Isis.WriteLine("WARNING: Isis Aggregation is discarding value from below with key=" + kvp.Value.key + " because of a timeout");
                                else if (kvp.Value.hasRValue && Isis.NOW() - kvp.Value.RValueTime > AggTimeout.when)
                                    Isis.WriteLine("WARNING: Isis Aggregation is discarding value from left with key=" + kvp.Value.key + " because of a timeout");
                                else
                                    nldValues.Add(kvp.Key.GetHashCode(), kvp.Value);
                            ldValues = nldValues;
                        }

                        len = toSend.Count();
                        vids = new int[len];
                        keys = new KeyType[len];
                        values = new ValueType[len];

                        foreach (AggInfo ai in myGroup.AggTypes)
                            if (ai.KT.Equals(typeof(KeyType)) && ai.VT.Equals(typeof(ValueType)))
                            {
                                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                                    Isis.WriteLine("Aggregator: GetAggState<" + typeof(KeyType) + ":" + typeof(ValueType) + ">, computed a vector of length " + toSend.Count() + " containing: ");
                                foreach (KeyValuePair<int, ldTuple<KeyType, ValueType>> kvp in toSend)
                                {
                                    keys[idx] = kvp.Value.key;
                                    vids[idx] = kvp.Value.vid;
                                    // Don't show the view id to the aggregator method; might confuse the developer
                                    long mylid = 0;
                                    if (myGroup.myLoggingFcn != null)
                                        myGroup.myLoggingFcn(Group.IL_AGGREGATE, Group.IL_START, Isis.my_address, mylid = myGroup.newLoggingId(), kvp.Value.key, kvp.Value.rValue, kvp.Value.dValue);
                                    values[idx] = theAggMethod(kvp.Value.key, kvp.Value.rValue, kvp.Value.dValue);
                                    if (myGroup.myLoggingFcn != null)
                                        myGroup.myLoggingFcn(Group.IL_AGGREGATE, Group.IL_DONE, Isis.my_address, mylid, values[idx]);
                                    if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                                        Isis.WriteLine("       { vid = " + vids[idx] + ", key=" + keys[idx] + ", value=" + values[idx] + " }");
                                    ++idx;
                                }
                                byte[] result = Msg.toBArray(idx, vids, keys, values);
                                return result;
                            }
                        throw new IsisException("IsisToken: Can't find Aggregate method for <" + typeof(KeyType) + "," + typeof(ValueType) + ">");

                    case BreakWaits:
                        BreakMyWaits(vid);
                        break;
                }
                return null;
            }

            /// <exclude></exclude>
            public string AggName()
            {
                return "IsisAggregator<" + typeof(KeyType) + ", " + typeof(ValueType) + ">";
            }

            internal class wTI
            {
                internal Thread who;
                internal KeyType key;

                internal wTI(Thread t, KeyType k)
                {
                    who = t;
                    key = k;
                }
            }
            internal List<wTI> WaitingThreads = new List<wTI>();

            /// <exclude></exclude>
            public string AggState()
            {
                string s = "";
                using (new LockAndElevate(Alock))
                {
                    foreach (wTI wti in WaitingThreads)
                        s += "            Thread " + (wti.who.Name == null ? "(no name)" : wti.who.Name) + " is waiting for Aggregation Result(Key=" + wti.key + ")\r\n";
                    foreach (KeyValuePair<int, ldTuple<KeyType, ValueType>> kvp in ldValues)
                    {
                        if (kvp.Value.hasDValue)
                            s += "            Value from below: (Key=" + kvp.Key + ", Value = " + kvp.Value.dValue + ", learned at " + Isis.TimeToString(kvp.Value.DValueTime) + ")\r\n";
                        if (kvp.Value.hasRValue)
                            s += "            Value from right: (Key=" + kvp.Key + ", Value = " + kvp.Value.rValue + ", learned at " + Isis.TimeToString(kvp.Value.RValueTime) + ")\r\n";
                    }
                }
                return s;
            }

            internal Object NULL = new Object();

            internal class ldTuple<KT, VT>
            {
                internal int vid;
                internal bool hasRValue;
                internal long RValueTime;
                internal bool hasDValue;
                internal long DValueTime;
                internal KT key;
                internal VT rValue;
                internal VT dValue;

                internal ldTuple(int v, KT k, Object lvalue, VT dvalue)
                {
                    vid = v;
                    key = k;
                    SetDValue(vid, dvalue);
                }

                internal void SetDValue(int v, VT dvalue)
                {
                    dValue = dvalue;
                    hasDValue = true;
                    DValueTime = Isis.NOW();
                    if (vid == 0)
                        vid = v;
                    else if (v != 0 && vid != v)
                        vid = -v;
                }
                internal ldTuple(int v, KT k, VT lvalue, Object dvalue)
                {
                    vid = v;
                    key = k;
                    SetRValue(vid, lvalue);
                }

                internal void SetRValue(int v, VT lvalue)
                {

                    rValue = lvalue;
                    hasRValue = true;
                    RValueTime = Isis.NOW();
                    if (vid == 0)
                        vid = v;
                    else if (v != 0 && vid != v)
                        vid = -v;
                }

            }

            internal Dictionary<int, ldTuple<KeyType, ValueType>> ldValues = new Dictionary<int, ldTuple<KeyType, ValueType>>(Group.tokenInfo.RINGSIZE * 2);

            internal void SetRValue(int vid, KeyType id, ValueType lValue)
            {
                SetRValue(0, vid, id, lValue);
            }

            internal void SetRValue(int level, int vid, KeyType id, ValueType lValue)
            {
                int hc = id.GetHashCode();
                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("Aggregation: SetRValue for id=" + id + "(hashcode=" + id.GetHashCode() + "), vid = " + vid + ", level=" + level + ", value=" + lValue);
                using (new LockAndElevate(Alock))
                {
                    ldTuple<KeyType, ValueType> ldp;
                    if (ldValues.TryGetValue(hc, out ldp))
                    {
                        detectCollisions(id, ldp.key);
                        ldp.SetRValue(vid, lValue);
                    }
                    else
                        ldValues.Add(hc, new ldTuple<KeyType, ValueType>(vid, id, lValue, NULL));
                }
                SendAggIfBothReady(level, vid, id);
            }

            internal void SetDValue(int vid, KeyType id, ValueType dValue)
            {
                SetDValue(0, vid, id, dValue);
            }

            internal void SetDValue(int level, int vid, KeyType id, ValueType dValue)
            {
                int hc = id.GetHashCode();
                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("Aggregation: SetDValue for id=" + id + ", level=" + level + ", value=" + dValue);
                using (new LockAndElevate(Alock))
                {
                    ldTuple<KeyType, ValueType> ldp;
                    if (ldValues.TryGetValue(hc, out ldp))
                    {
                        detectCollisions(id, ldp.key);
                        ldp.SetDValue(vid, dValue);
                    }
                    else
                        ldValues.Add(hc, new ldTuple<KeyType, ValueType>(vid, id, NULL, dValue));
                }
                SendAggIfBothReady(level, vid, id);
            }

            private void detectCollisions(KeyType k1, KeyType k2)
            {
                if (k1.Equals(k2))
                    return;
                throw new IsisException("Keys " + k1 + " and " + k2 + " have same HashCode but aren't equal!");
            }

            internal void gotRValues(byte[] ba)
            {
                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("Aggregation: gotRValues");
                Object[] obs = Msg.BArrayToObjects(ba, typeof(int[]), typeof(KeyType[]), typeof(ValueType[]));
                int[] vids = (int[])obs[0];
                KeyType[] ids = (KeyType[])obs[1];
                ValueType[] aggs = (ValueType[])obs[2];
                for (int idx = 0; idx < ids.Length; idx++)
                    SetRValue(vids[idx], ids[idx], aggs[idx]);
            }

            internal void gotDValues(byte[] ba)
            {
                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("Aggregation: gotDValues");
                Object[] obs = Msg.BArrayToObjects(ba, typeof(int[]), typeof(KeyType[]), typeof(ValueType[]));
                int[] vids = (int[])obs[0];
                KeyType[] ids = (KeyType[])obs[1];
                ValueType[] aggs = (ValueType[])obs[2];
                for (int idx = 0; idx < ids.Length; idx++)
                    SetDValue(vids[idx], ids[idx], aggs[idx]);
            }

            internal void gotMyAggregates(byte[] ba)
            {
                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("Aggregation: gotMyAggregates");
                Object[] obs = Msg.BArrayToObjects(ba, typeof(int[]), typeof(KeyType[]), typeof(ValueType[]));
                int[] vids = (int[])obs[0];
                KeyType[] ids = (KeyType[])obs[1];
                ValueType[] aggs = (ValueType[])obs[2];
                for (int idx = 0; idx < ids.Length; idx++)
                    SetAggResult(vids[idx], ids[idx], aggs[idx]);
            }

            internal void SetAggResult(int vid, KeyType key, ValueType res)
            {
                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("Aggregation: SetAggResult for id=" + key + ", level=" + myLevel + ", value=" + res);
                Monitor.Enter(AggResult.theBarrier);
                AggResult.saveResult(key, res);
                Monitor.PulseAll(AggResult.theBarrier);
                Monitor.Exit(AggResult.theBarrier);
            }

            internal void SendAggIfBothReady(int level, int vid, KeyType key)
            {
                if ((myGroup.flags & Group.G_ISLARGE) != 0)
                    return;
                View theView;
                using (new LockAndElevate(myGroup.ViewLock))
                    theView = myGroup.theView;
                Address[] participants = Participants(key, theView);
                int myRank = 0;
                while (myRank < participants.Length && !participants[myRank].isMyAddress())
                    myRank++;
                if (myRank == participants.Length)
                    throw new IsisException("SendAggIfBothReady: I'm not in participant list!");
                using (new LockAndElevate(Alock))
                {
                    ldTuple<KeyType, ValueType> ldp;
                    int id = key.GetHashCode();
                    if (ldValues.TryGetValue(id, out ldp))
                    {
                        if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                            Isis.WriteLine(".... SendAggIfBothReady at level " + level + ", vid=" + vid + ", key=" + key + ", IAmRightMost=" + IAmRightMost(key, level, participants) + ", ldp.hasDValue=" + ldp.hasDValue + ", ldp.hasRValue=" + ldp.hasRValue);
                        if (IAmRightMost(key, level, participants) && ldp.hasDValue)
                        {
                            level = AggLevelToSendAt(key, level, theView, myRank, participants);
                            if (level > myGroup.AggList.Length - 1)
                            {
                                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                                    Isis.WriteLine("SendAggIfBothReady<" + myGroup.gname + "> finalizing(1) value for key " + key + ", value " + ldp.dValue);
                                SetAggResult(vid, key, ldp.dValue);
                            }
                            else
                            {
                                ValueType value = ldp.hasRValue ? theAggMethod(key, ldp.rValue, ldp.dValue) : ldp.dValue;
                                AggSendToNextOnLevel(level, vid, participants, key, value, theView, level, myRank, myRank);
                            }
                            ldValues.Remove(id);
                        }
                        else if (ldp.hasDValue && ldp.hasRValue)
                        {
                            int target = myRank & ~(1 << level);
                            level = AggLevelToSendAt(key, level + 1, theView, target, participants);
                            if (level > myGroup.AggList.Length - 1)
                            {
                                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                                    Isis.WriteLine("SendAggIfBothReady<" + myGroup.gname + "> finalizing(2) value for key " + key + ", value " + ldp.dValue);
                                SetAggResult(vid, key, ldp.dValue);
                            }
                            else
                                AggSendToNextOnLevel(level, vid, participants, key, theAggMethod(key, ldp.rValue, ldp.dValue), theView, level, myRank, target);
                            ldValues.Remove(id);
                        }

                        else if (ldp.hasRValue && level == myGroup.AggList.Length - 1)
                        {
                            if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                                Isis.WriteLine("SendAggIfBothReady<" + myGroup.gname + "> finalizing(3) value for key " + key + ", value " + ldp.rValue);
                            SetAggResult(vid, key, ldp.rValue);
                            ldValues.Remove(id);
                        }
                    }
                }
            }

            internal bool IAmRightMost(KeyType key, int level)
            {
                View theView;
                using (new LockAndElevate(myGroup.ViewLock))
                    theView = myGroup.theView;
                return IAmRightMost(key, level, Participants(key, theView));
            }

            internal bool IAmRightMost(KeyType key, int level, Address[] participants)
            {
                int rank = FindMe(key, participants);
                return ((rank >> level) & 1) != 0 || (rank + (1 << level) >= participants.Length);
            }

            internal bool IAmLeftMost(KeyType key, int level, Address[] participants)
            {
                return ((FindMe(key, participants) >> level) & 1) == 0;
            }

            private int FindMe(KeyType key)
            {
                return FindMe(key, null);
            }

            private int FindMe(KeyType key, Address[] participants)
            {
                if (participants == null)
                {
                    View theView;
                    using (new LockAndElevate(myGroup.ViewLock))
                        theView = myGroup.theView;
                    participants = Participants(key, theView);
                }
                int myRank = 0;
                while (myRank != participants.Length)
                    if (participants[myRank].isMyAddress())
                        break;
                    else
                        ++myRank;
                if (myRank == participants.Length)
                    throw new IsisException("FindMe");
                return myRank;
            }

            private int AggLevelToSendAt(KeyType key, int level, View v, int rank, Address[] participants)
            {
                while (level < myGroup.AggList.Length - 1 && ((rank >> level) & 1) == 0 && (rank + (1 << level)) >= participants.Length)
                    ++level;
                return level;
            }

            private void AggSendToNextOnLevel(int level, int vid, Address[] participants, KeyType key, ValueType value, View theView, int curLevel, int myRank, int targetRank)
            {
                AggSendToNextOnLevel(level, vid, participants, key, value, theView, curLevel, myRank, targetRank, true);
            }

            // Sends to the node to the left (the "next" node) of the target, or to the target if there is no node to its left
            private void AggSendToNextOnLevel(int level, int vid, Address[] participants, KeyType key, ValueType value, View theView, int curLevel, int myRank, int targetRank, bool doSend)
            {
                targetRank &= ~(1 << level);
                Address next;
                next = (targetRank >= participants.Length) ? participants[0] : participants[targetRank];
                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("AggSendToNextOnLevel<" + myGroup.gname + ", key=" + key + ", value=" + value + "> currently level " + curLevel + ", sending  at level " + level +
                        ((myRank >= targetRank) ? ", DValue" : ", RValue") + " to target=" + targetRank + ", id=" + next + " with participants=" + Address.VectorToString(participants));
                if (doSend)
                    myGroup.P2PSend(next, Isis.SGAGGREGATE, Isis.my_address, myRank > targetRank, level, vid, Msg.toBArray(key, value));
            }

            /// <exclude>
            /// <summary>
            /// Used for small group aggregation
            /// </summary>
            /// <param name="level"></param>
            /// <param name="vid"></param>
            /// <param name="ko"></param>
            /// <param name="vo"></param>
            /// <param name="fromBelow"></param>
            /// </exclude>
            public void GotSGAggInfo(bool fromBelow, int level, int vid, object ko, object vo)
            {
                KeyType key = (KeyType)ko;
                ValueType value = (ValueType)vo;
                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("Aggregation: GotSAggInfo: fromBelow=" + fromBelow + ", level=" + level + ", vid=" + vid + ", key=" + key + ", value=" + value);
                if (IAmRightMost(key, level) || fromBelow)
                    SetRValue(level, vid, key, value);
                else
                    SetDValue(level, vid, key, value);
                SendAggIfBothReady(level, vid, key);
            }

            volatile internal bool unwinding = false;

            internal void BreakMyWaits(int vid)
            {
                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("Aggregation: Unwinding waits in aggregator for vid=" + myVid + " prior to view-driven reinitialization vid=" + vid);
                Monitor.Enter(AggResult.theBarrier);
                // Once unwinding is set to true, we're in the midst of discarding this entire
                // aggregation object and creating a new one for the next group view.
                unwinding = true;
                Monitor.PulseAll(AggResult.theBarrier);
                Monitor.Exit(AggResult.theBarrier);
            }

            internal void Set(int vid, KeyType id, ValueType value)
            {
                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                    Isis.WriteLine("Aggregation: Set for id=" + id + ", value=" + value);
                Address[] participants;
                using (new LockAndElevate(myGroup.ViewLock))
                    participants = Participants(id, myGroup.theView);
                if (participants.Length == 1)
                {
                    // Special situation: I'm the only member
                    SetAggResult(vid, id, value);
                    return;
                }
                SetDValue(vid, id, value);
            }

            internal ValueType Get(KeyType id)
            {
                wTI wti = new wTI(Thread.CurrentThread, id);
                ValueType v = default(ValueType);
                try
                {
                    using (new LockAndElevate(Alock))
                        WaitingThreads.Add(wti);
                    if (!IsisSystem.IsisActive)
                        throw new IsisException("Isis inactive");
                    if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                        Isis.WriteLine("Aggregation: Entry to get for id=" + id);
                    Monitor.Enter(AggResult.theBarrier);
                    while (!unwinding && IsisSystem.IsisActive)
                    {
                        object theResult = AggResult.getResult(id);
                        if (theResult == null)
                        {
                            v = default(ValueType);
                            if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                                Isis.WriteLine("Await Aggregation result for id=" + id);
                            if (Monitor.Wait(AggResult.theBarrier, AggTimeout.when) == false)
                                break;
                        }
                        else
                        {
                            v = (ValueType)theResult;
                            break;
                        }
                    }
                    Monitor.Exit(AggResult.theBarrier);
                    if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                        Isis.WriteLine("Aggregation: Get for id=" + id + " will return " + ((unwinding || v == null) ? "AggregationFailed exception or timeout" : v.ToString()));
                    if (unwinding || v == null)
                        throw new AggregationFailed(1);
                }
                finally
                {
                    using (new LockAndElevate(Alock))
                        WaitingThreads.Remove(wti);
                }
                return v;
            }
        }

        /* A note about the OOB logic.  This code actually always (only) runs in the Isis.ISISMEMBERS group.  However, it can be called
         * from other groups, and when that is done, it records the associated gaddr.  The encipher/decipher logic then uses that gaddr
         * for encryption purposes.  But there really is just one OOB layer for the system, running in ISISMEMBERS, and this is why I've
         * declared the Map and Lock objects static -- I wanted to underscore the reasoning so that correctness would be more clear.
         */
        internal class OOBInfo
        {
            internal volatile int flags;
            internal string OOBFname;
            internal Address gaddr;
            internal MemoryMappedFile mmf;
            internal MemoryMappedViewAccessor mmva;
            internal List<Address> replicas;
            internal List<List<Address>> newCopies = new List<List<Address>>();
            internal List<Address> currentlySendingTo;
            internal bool[] OOBStatus;
            internal Semaphore mmfReady = new Semaphore(0, int.MaxValue);
            public Action<string, MemoryMappedFile> WhenDone;

            internal const int OBI_MASTER = 0x0001;
            internal const int OBI_FETCHWAIT = 0x0002;                   // Waiting in OOBFetch
            internal const int OBI_SNDWAIT = 0x0004;
            internal const int OBI_RCVWAIT = 0x0008;
            internal const int OBI_DELETE = 0x0010;
            internal const int OBI_ACTIVE = 0x0020;
            internal const int OBI_RCVING = 0x0040;
            internal const int OBI_SENDING = 0x0080;
            internal const int OBI_INITIALVIEW = 0x0100;

            internal static string[] fnames = new string[] { "MASTER", "Fetch-wait", "Send-wait", "Recv-wait", "DELETE", "XFER-SCHEDULED", "Recieving", "Sending", "xfer-initialview" };

            internal OOBInfo(string OOBFname)
                : this(OOBFname, false, null, new List<Address>(), null)
            {
            }

            internal OOBInfo(string OOBFname, bool ready, MemoryMappedFile mmf, List<Address> replicas, Address gaddr)
            {
                this.OOBFname = OOBFname;
                this.gaddr = gaddr;
                this.replicas = replicas;
                if ((this.mmf = mmf) != null)
                {
                    mmva = mmf.CreateViewAccessor();
                    OOBStatus = new bool[(int)((mmva.Capacity + Isis.ISIS_OOBCHUNKSIZE) / Isis.ISIS_OOBCHUNKSIZE)];
                }
                if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                    Isis.WriteLine("Creating a new OOBInfo object: " + OOBFname + ", replicas at " + Address.VectorToString(replicas.ToArray()));
            }

            // Caller must hold OOBMapLock
            public override string ToString()
            {
                string whenDone = WhenDone == null ? "(no action)" : "upcall to user-defined method";
                string fstr = " ";
                for (int b = 0; (1 << b) <= flags; b++)
                    if ((flags & (1 << b)) != 0)
                        fstr += fnames[b] + " ";
                string bvec = ", block-status=";
                if (OOBStatus != null)
                    foreach (var b in OOBStatus)
                        bvec += b ? "1" : "0";
                return "    [" + OOBFname + "]: " + ((mmf != null) ? "sender" : "receiver") + (fstr.Length > 1 ? ", flags=[" + fstr + "]" : "") + (OOBStatus != null ? bvec : "") +
                    ", replicas=" + Address.VectorToString(replicas.ToArray()) + ", when done: " + whenDone;
            }
        }

        internal static Dictionary<string, OOBInfo> OOBMap = new Dictionary<string, OOBInfo>(1000);
        internal static LockObject OOBMapLock = new LockObject("OOBMapLock");


        /// <summary>
        /// Used to create a new OutOfBand object representing the designated memory-mapped file.  Called only once,
        /// by the first group member to map the object.
        /// </summary>
        /// <param name="OOBFname">The file name</param>
        /// <param name="mmf">The memory mapped file handle</param>
        public void OOBRegister(string OOBFname, MemoryMappedFile mmf)
        {
            OOBRegister(OOBFname, gaddr, mmf);
        }

        private void OOBRegister(string OOBFname, Address gaddr, MemoryMappedFile mmf)
        {
            flags |= G_USESOOB;
            if (this != Isis.ISISMEMBERS)
            {
                Isis.ISISMEMBERS.OOBRegister(OOBFname, gaddr, mmf);
                return;
            }
            OOBRegister(false, OOBFname, gaddr, mmf, new List<Address>() { Isis.my_address });
        }

        /// <summary>
        /// Register a memory-mapped ax-file and a list of locations where copies exist.  Called only once, by a member.
        /// MMf should be null if the caller itself isn't in the "where" list
        /// </summary>
        /// <param name="OOBFname">The file name</param>
        /// <param name="mmf">The memory mapped object</param>
        /// <param name="where">Where copies can be found</param>
        public void OOBRegister(string OOBFname, MemoryMappedFile mmf, List<Address> where)
        {
            OOBRegister(false, OOBFname, gaddr, mmf, where);
        }

        internal void OOBRegister(bool registerSpecial, string OOBFname, Address gaddr, MemoryMappedFile mmf, List<Address> where)
        {
            flags |= G_USESOOB;
            if (this != Isis.ISISMEMBERS)
            {
                Isis.ISISMEMBERS.OOBRegister(false, OOBFname, gaddr, mmf, where);
                return;
            }
            using (new LockAndElevate(OOBMapLock))
            {
                if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                    Isis.WriteLine("OOBUpdateRegistry:\r\n" + OOBState(null));
                OOBInfo obi = null;
                if (!OOBMap.TryGetValue(OOBFname, out obi))
                {
                    if (where.Count() > 0)
                        OOBMap.Add(OOBFname, obi = new OOBInfo(OOBFname, mmf != null, mmf, where, gaddr));
                }
                else
                {
                    if (where.Count() == 0)
                    {
                        if (obi.mmf != null)
                        {
                            obi.mmf.Dispose();
                            obi.mmva.Dispose();
                        }
                        if ((obi.flags & ~(OOBInfo.OBI_MASTER | OOBInfo.OBI_ACTIVE)) != 0)
                            throw new IsisException("OOBDelete when transfer was active for " + obi);
                        OOBMap.Remove(OOBFname);
                        if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                            Isis.WriteLine("Delete existing OOBInfo object: " + OOBFname);
                    }
                    else
                    {
                        foreach (Address w in where)
                            if (!obi.replicas.Contains(w))
                                obi.replicas.Add(w);
                    }
                }
                if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                    Isis.WriteLine("Update existing OOBInfo object: " + OOBFname + " set MASTER mode, release mmf-ready semaphore, replicas at " + Address.VectorToString(obi.replicas.ToArray()));
                obi.flags |= OOBInfo.OBI_MASTER;
                obi.mmfReady.Release(1);
            }
            if (!registerSpecial)
                OrderedSend(Isis.OUTOFBAND, OOBFname, gaddr, where);
            else // Special for initial view state transfer only
                doOOBUpdateRegistry(OOBFname, gaddr, where);
        }

        /// <summary>
        /// OOB subsystem will do an upcall to this method when a new OOB file is received, if the method is specified.
        /// </summary>
        /// <param name="OOBFname">Filename of the received OOB file</param>
        /// <param name="mmf">Memory mapped file handle for it</param>
        public delegate void OOBCompletionNotifier(string OOBFname, MemoryMappedFile mmf);

        internal static OOBCompletionNotifier theNotifier;

        /// <summary>
        /// OOB subsystem will do an upcall to this method when a new OOB file is received, if the method is specified.
        /// </summary>
        /// <param name="tn">The method to call</param>
        public static void OOBSetCompletionNotifier(OOBCompletionNotifier tn)
        {
            theNotifier = tn;
        }

        /// <summary>
        /// Modify the replication pattern for the designated object using the new list of desired locations
        /// </summary>
        /// <param name="OOBFname">File name</param>
        /// <param name="where">New pattern</param>
        public void OOBReReplicate(string OOBFname, List<Address> where)
        {
            OOBReReplicate(OOBFname, where, null);
        }

        /// <summary>
        /// Modify the replication pattern for the designated object using the new list of desired locations
        /// </summary>
        /// <param name="OOBFname">File name</param>
        /// <param name="where">New pattern</param>
        /// <param name="whenDone">Callback when rereplication is finished</param>
        public void OOBReReplicate(string OOBFname, List<Address> where, Action<string, MemoryMappedFile> whenDone)
        {
            OOBReReplicate(OOBFname, where, whenDone, gaddr);
        }

        private void OOBReReplicate(string OOBFname, List<Address> where, Action<string, MemoryMappedFile> whenDone, Address gaddr)
        {
            if (this != Isis.ISISMEMBERS)
            {
                Isis.ISISMEMBERS.OOBReReplicate(OOBFname, where, whenDone, gaddr);
                return;
            }
            OOBReReplicate(new List<OOBRepInfo>() { new OOBRepInfo(OOBFname, gaddr, where) }, whenDone);
        }

        /// <summary>
        /// Modify the replication pattern for the designated objects using the new list of desired locations
        /// </summary>
        /// <param name="repInfo">Summary of a set of desired rereplication actions</param>
        /// <param name="whenDone">Callback when rereplication finishes</param>
        public void OOBReReplicate(List<OOBRepInfo> repInfo, Action<string, MemoryMappedFile> whenDone)
        {
            OOBReReplicate(false, repInfo, whenDone);
        }

        internal void OOBReReplicate(bool registerSpecial, List<OOBRepInfo> repInfo, Action<string, MemoryMappedFile> whenDone)
        {
            if (this != Isis.ISISMEMBERS)
            {
                Isis.ISISMEMBERS.OOBReReplicate(repInfo, whenDone);
                return;
            }
            foreach (var ri in repInfo)
            {
                using (new LockAndElevate(OOBMapLock))
                {
                    OOBInfo obi = null;
                    if (OOBMap.TryGetValue(ri.OOBFname, out obi))
                    {
                        if (ri.OOBReplicas.Count() > 0 && ri.OOBReplicas.Intersect(obi.replicas).Count() == 0)
                            throw new IsisException("OOBReReplicate: New replicas " + Address.VectorToString(ri.OOBReplicas.ToArray()) + " are required to overlap with old replicas " +
                                Address.VectorToString(obi.replicas.ToArray()) + "(consider splitting the request into two subrequests)");
                    }
                    else
                        throw new IsisException("OOBReReplicate: File name " + ri.OOBFname + " must be registered before ReReplication is requested");
                    if (registerSpecial)
                        obi.flags |= OOBInfo.OBI_INITIALVIEW;
                    obi.WhenDone = whenDone;
                }
            }
            if (!registerSpecial)
                OrderedSend(Isis.OUTOFBAND, repInfo);
            else // Special for state transfer of initial view
                doOOBUpdateRegistry(repInfo);
        }

        internal static int OOBInUse;
        internal static List<OOBInfo> OOBWaitList = new List<OOBInfo>();

        internal void doOOBUpdateRegistry(string OOBFname, Address gaddr, List<Address> where)
        {
            doOOBUpdateRegistry(new List<OOBRepInfo>() { new OOBRepInfo(OOBFname, gaddr, where) });
        }

        internal void doOOBUpdateRegistry(List<OOBRepInfo> repInfo)
        {
            flags |= G_USESOOB;
            foreach (var ri in repInfo)
            {
                bool createdNew = false;
                List<Address> newCopies = new List<Address>();
                List<Address> unNeeded = new List<Address>();
                OOBInfo obi;
                using (new LockAndElevate(OOBMapLock))
                {
                    if (!OOBMap.TryGetValue(ri.OOBFname, out obi))
                    {
                        if (ri.OOBReplicas.Count() == 0)
                            continue;
                        obi = new OOBInfo(ri.OOBFname, false, null, ri.OOBReplicas, ri.gaddr);
                        OOBMap.Add(ri.OOBFname, obi);
                        createdNew = true;

                    }
                    else if (ri.OOBReplicas.Count() == 0)
                    {
                        OOBRemoveObi(obi);
                        continue;
                    }
                    else
                    {
                        if (obi.replicas.Count() == 0)
                            createdNew = true;
                    }
                    if (!createdNew)
                    {
                        foreach (Address w in ri.OOBReplicas)
                            if (!obi.replicas.Contains(w))
                                newCopies.Add(w);
                        foreach (Address w in obi.replicas)
                            if (!ri.OOBReplicas.Contains(w))
                                unNeeded.Add(w);
                    }
                    obi.replicas = ri.OOBReplicas;
                    if (obi.mmf != null)
                    {
                        if (unNeeded.Contains(Isis.my_address))
                            obi.mmf = null;
                    }
                    if (newCopies.Count() > 0)
                    {
                        obi.newCopies.Add(newCopies);
                        OOBWaitList.Add(obi);
                    }
                    if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                        Isis.WriteLine(OOBState(null));
                }
            }
            OOBCheckQ();
        }

        private void OOBRemoveObi(OOBInfo obi)
        {
            if ((obi.flags & OOBInfo.OBI_FETCHWAIT) != 0)
            {
                if ((obi.flags & ~(OOBInfo.OBI_MASTER | OOBInfo.OBI_ACTIVE)) != 0)
                    Isis.WriteLine("doOOBUpdateRegistry/OOBDelete: FETCHWAIT was active (toggle semaphore for " + obi.OOBFname + ")");
                obi.mmfReady.Release(1);
                obi.flags &= ~OOBInfo.OBI_FETCHWAIT;
            }
            if ((obi.flags & ~(OOBInfo.OBI_MASTER | OOBInfo.OBI_ACTIVE | OOBInfo.OBI_INITIALVIEW)) != 0)
                Isis.WriteLine("WARNING: OOBDelete was issued while OOB transfer was still active for " + obi);
            OOBMap.Remove(obi.OOBFname);
            if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                Isis.WriteLine("doOOBUpdateRegistry/Delete existing OOBInfo object: " + obi.OOBFname);
        }

        internal static string OOBState(Group g)
        {
            string s = "";
            if (g == Isis.ISISMEMBERS || g == null)
                using (new LockAndElevate(OOBMapLock))
                {
                    s += "OOBWaitList: { ";
                    foreach (var obi in OOBWaitList)
                        s += obi.OOBFname + " ";
                    s += "}\r\n";
                    foreach (var kvp in OOBMap)
                        s += kvp.Value + "\r\n";
                }
            else
            {
                using (new LockAndElevate(OOBMapLock))
                {
                    bool matched = false;
                    s += "OOB files associated with this group: { ";
                    foreach (var kvp in OOBMap)
                        if (kvp.Value.gaddr.Equals(g.gaddr))
                        {
                            matched = true;
                            s += kvp.Value.OOBFname + " ";
                        }
                    if (!matched)
                        return "";
                    s += "}\r\n";
                }
            }
            return s;
        }

        private const int CHUNKSPERTRY = 75;
        private const int LOWATERMARK = 20;

        List<byte[]> RecycledBuffers = new List<byte[]>();

        // Called from someone who received an OrderedSend
        internal void OOBCheckQ()
        {
            OOBCheckQ(null);
        }

        internal delegate int qmethod(int nr, params object[] obs);

        // Called from someone who received an OrderedSend
        internal static void OOBCheckQ(OOBInfo obi)
        {
            using (new LockAndElevate(OOBMapLock))
            {
                if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                {
                    string allofem = " ";
                    foreach (var oobi in OOBWaitList)
                        allofem += oobi.OOBFname + " ";
                    if (allofem.Length > 1)
                        allofem = "all; waitlist={" + allofem + "}, waitlist.Count()=" + OOBWaitList.Count();
                    Isis.WriteLine("**** Entered checkOOBQ[" + (obi == null ? allofem : obi.OOBFname) + "]: OOBInUse=" + OOBInUse +
                        " (limit is " + (Isis.OOBMAXIPMCADDRS - 1) + "), WaitList.Count()=" + OOBWaitList.Count() + "\r\n" + OOBState(null));
                }
                if (OOBInUse == Isis.OOBMAXIPMCADDRS - 1 || OOBWaitList.Count() == 0)
                    return;
                List<Address> newCopies;
                if (obi == null)
                {
                    int which, nWaiting = OOBWaitList.Count();
                    bool overlapsActiveXfer = true;
                    for (which = 0; which < nWaiting; which++)
                    {
                        overlapsActiveXfer = false;
                        obi = OOBWaitList[which];
                        if ((obi.flags & OOBInfo.OBI_ACTIVE) != 0)
                            continue;
                        foreach (var oobi in OOBMap)
                            if ((oobi.Value.flags & OOBInfo.OBI_ACTIVE) != 0 &&
                                (oobi.Value.currentlySendingTo != null && new HashSet<Address>(oobi.Value.currentlySendingTo).Overlaps(obi.newCopies[0])))
                            {
                                overlapsActiveXfer = true;
                                break;
                            }
                        if (!overlapsActiveXfer)
                            break;
                    }
                    if (which == nWaiting)
                        return;
                    if ((obi.flags & OOBInfo.OBI_ACTIVE) != 0)
                        throw new IsisException("OOBCheckQ: obi[which=" + which + ", nWaiting=" + nWaiting + "] was already active! " + obi);
                    OOBWaitList.Remove(obi);
                    newCopies = obi.newCopies[0];
                    obi.currentlySendingTo = newCopies;
                    obi.newCopies.Remove(newCopies);
                    ++OOBInUse;
                }
                else
                    newCopies = obi.currentlySendingTo;
                if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                    Isis.WriteLine("**** checkOOBQ setting ACTIVE flag for " + obi.OOBFname + "... newCopies=" + Address.VectorToString(newCopies.ToArray()));
                obi.flags |= OOBInfo.OBI_ACTIVE;
                if (!obi.replicas.Contains(Isis.my_address))
                    return;
                if (newCopies.Contains(Isis.my_address) || !obi.replicas.Contains(Isis.my_address) || !obi.replicas[0].isMyAddress() || (obi.flags & OOBInfo.OBI_SENDING) != 0)
                {
                    if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                        Isis.WriteLine("**** checkOOBQ: Decided to take no action: newCopies.Contains(Isis.my_address)=" + newCopies.Contains(Isis.my_address) + " || !obi.replicas.Contains(Isis.my_address)=" +
                            obi.replicas.Contains(Isis.my_address) + "|| !obi.replicas[0].isMyAddress()=" + !obi.replicas[0].isMyAddress());
                    return;
                }
                if (obi.mmf != null)
                {
                    obi.flags |= OOBInfo.OBI_SENDING;
                    Thread t = new Thread(delegate()
                    {
                        int nchunks = (int)((OOBGetLength(obi.mmva) + Isis.ISIS_OOBCHUNKSIZE - 1) / Isis.ISIS_OOBCHUNKSIZE);
                        if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                            Isis.WriteLine("Initiating OOBXFER for " + obi.OOBFname + "(nchunks=" + nchunks + ")");
                        bool[] chunkAcked = new bool[nchunks];
                        int ipmcaddr = 0;
                        string mcaddr = null;
                        IPAddress physIP = null;
                        IPEndPoint remoteEP = null;
                        Socket sendSock = null;
                        int allAckedTo = 0;
                        qmethod theMethod = ((obi.flags & OOBInfo.OBI_INITIALVIEW) != 0) ? (qmethod)Isis.ISISMEMBERS.doPlatformLevelQuery : (qmethod)Isis.ISISMEMBERS.Query;
                        if (!Isis.ISIS_TCP_ONLY && !Isis.ISIS_UNICAST_ONLY)
                        {
                            ipmcaddr = MCMDSocket.AllocateAddr();
                            mcaddr = MCMDSocket.PMCAddr(ipmcaddr);
                            List<bool[]> outcomes = new List<bool[]>();
                            theMethod(ALL, new Timeout(1000000, Timeout.TO_FAILURE), newCopies, Isis.OUTOFBAND, mcaddr, obi.OOBFname, obi.gaddr, nchunks, OOBGetLength(obi.mmva), obi.replicas, EOL, outcomes);
                            chunkAcked = OOBUpdateStatus(obi, nchunks, ref allAckedTo, outcomes);
                            physIP = Isis.LastIPv4(mcaddr);
                            remoteEP = new IPEndPoint(physIP, Isis.ISIS_GROUPPORT);
                            sendSock = MCMDSocket.MCMDSockSetup(physIP);
                        }
                        while (chunkAcked != null && chunkAcked.Contains(false) && IsisSystem.IsisActive)
                        {
                            if (Isis.ISIS_TCP_ONLY || Isis.ISIS_UNICAST_ONLY)
                            {
                                // When multicast isn't available, we just send the data in chunks
                                int fives = nchunks / 5, cn = allAckedTo + 1;
                                for (int n = allAckedTo / 5; n < fives; n++)
                                {
                                    for (int k = 0; k < 5; k++)
                                    {
                                        if (!chunkAcked[cn])
                                            Isis.ISISMEMBERS.doSend(false, false, newCopies, Isis.OUTOFBAND, obi.OOBFname, OOBGetOffset(cn), OOBGetChunk(obi.mmva, cn));
                                        ++cn;
                                    }
                                    // Now pause until these have been sent
                                    Isis.ISISMEMBERS.Flush();
                                }
                            }
                            else
                            {
                                // Logic for the true multicast-based OOB send
                                int toSend = 0;
                                for (int n = nchunks - 1; n >= allAckedTo && toSend < CHUNKSPERTRY; --n)
                                    if (!chunkAcked[n])
                                        ++toSend;
                                try
                                {
                                    Semaphore DoneSema = new Semaphore(LOWATERMARK, int.MaxValue);
                                    int hashcode = obi.OOBFname.GetHashCode();
                                    for (int n = allAckedTo; n < nchunks && toSend > 0; n++)
                                        if (!chunkAcked[n])
                                        {
                                            obi.flags |= OOBInfo.OBI_SNDWAIT;
                                            DoneSema.WaitOne();
                                            obi.flags &= ~OOBInfo.OBI_SNDWAIT;
                                            byte[] buffer = Msg.toBArray(hashcode, OOBGetOffset(n), OOBGetChunk(obi.mmva, n));
                                            Group g = Group.doLookup(obi.gaddr);
                                            if (g != null && g.myAes != null)
                                                buffer = g.cipherBuf(buffer);
                                            if (buffer.Length > Isis.ISIS_MAXMSGLEN)
                                                throw new IsisException("OOB: Generated an object of length " + buffer.Length + ", but limit is " + Isis.ISIS_MAXMSGLEN);
                                            SocketAsyncEventArgs ae = new SocketAsyncEventArgs();
                                            ae.SetBuffer(buffer, 0, buffer.Length);
                                            ae.RemoteEndPoint = remoteEP;
                                            ae.SendPacketsSendSize = (int)Isis.ISIS_MAXMSGLEN;
                                            ae.UserToken = DoneSema;
                                            ae.Completed += AsyncSendDone;
                                            if (!sendSock.SendToAsync(ae))
                                                AsyncSendDone(null, ae);
                                            --toSend;
                                        }
                                    for (int n = 0; n < LOWATERMARK; n++)
                                        DoneSema.WaitOne();
                                }
                                catch (SocketException e)
                                {
                                    Isis.WriteLine("CheckOOBQ: Async sender thread threw a socket exception with inner code: " + e.ErrorCode);
                                }
                                catch (ObjectDisposedException)
                                {
                                }
                            }
                            List<bool[]> outcomes = new List<bool[]>();
                            theMethod(ALL, new Timeout(1000000, Timeout.TO_FAILURE), newCopies, Isis.OUTOFBAND, mcaddr, obi.OOBFname, obi.gaddr, nchunks, OOBGetLength(obi.mmva), obi.replicas, EOL, outcomes);
                            chunkAcked = OOBUpdateStatus(obi, nchunks, ref allAckedTo, outcomes);
                            if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                            {
                                string ostr = "";
                                foreach (var oc in outcomes)
                                {
                                    ostr += "\r\n---         [";
                                    for (int b = 0; b < chunkAcked.Length; b++)
                                        ostr += oc[b] ? "1" : "0";
                                    ostr += "]";
                                }
                                Isis.WriteLine("Sender for " + obi.OOBFname + " expected " + newCopies.Count() + " replies, got " + outcomes.Count() + "; allAckedTo=" + allAckedTo + ", bit-maps=" + ostr);
                            }
                        }
                        if (sendSock != null)
                            sendSock.Close();
                        MCMDSocket.DeAllocateAddr(ipmcaddr);
                        if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                            Isis.WriteLine("Successful completion of OOBXFER for " + obi.OOBFname);
                        obi.flags &= ~OOBInfo.OBI_SENDING;
                        Isis.ISISMEMBERS.OrderedQuery(ALL, new Timeout(1000000, Timeout.TO_FAILURE), Isis.OUTOFBAND, obi.OOBFname, new EOLMarker());
                        using (new LockAndElevate(OOBMapLock))
                            if (obi.WhenDone != null)
                            {
                                obi.WhenDone(obi.OOBFname, obi.mmf);
                                obi.WhenDone = null;
                            }
                    });
                    t.Name = "OOB file send thread for " + obi.OOBFname;

                    t.Start();
                }
                else
                    throw new IsisException("OOB: My role to send, but mmf=null!");
            }
        }

        static int counter;

        // Used only for state transfer of initial view to a joining member
        private int doPlatformLevelQuery(int nreplies, params object[] given)
        {
            Timeout timeout;
            object[] resRefs;
            splitObs(this, out timeout, ref given, out resRefs);
            List<Address> dests = new List<Address>();
            foreach (Address dest in (List<Address>)given[0])
                if (!dest.isMyAddress())
                    dests.Add(dest);
            given = fixObs(given);
            nreplies = dests.Count();
            cbCheck(given);
            Msg m = new Msg(given);
            m.vid = 0;
            m.msgid = ++counter;
            m.flags |= Msg.NEEDSREPLY;
            List<byte[]> barrays = _Query(m.vid, m.msgid, 0, false, false, (querySender)delegate(object[] obs)
            {
                foreach (Address dest in dests)
                    doPureP2PSend(dest, true, obs[0]);
            }, nreplies, timeout, new object[] { m });
            Msg.BArraysToLists(resRefs, barrays);
            return barrays.Count();
        }

        private static bool[] OOBUpdateStatus(OOBInfo obi, int nchunks, ref int allAckedTo, List<bool[]> outcomes)
        {
            bool[] chunkAcked;
            chunkAcked = null;
            foreach (var oc in outcomes)
                if (chunkAcked == null)
                    chunkAcked = oc;
                else for (int b = 0; b < chunkAcked.Length; b++)
                        chunkAcked[b] &= oc[b];
            while (allAckedTo < nchunks && chunkAcked[allAckedTo])
                ++allAckedTo;
            obi.OOBStatus = chunkAcked;
            return chunkAcked;
        }

        private static void AsyncSendDone(object sender, SocketAsyncEventArgs ae)
        {
            ((Semaphore)ae.UserToken).Release(1);
            ae.SetBuffer(null, 0, 0);
        }

        private void doOOBDone(string OOBFname)
        {
            OOBInfo obi;
            using (new LockAndElevate(OOBMapLock))
            {
                if (!OOBMap.TryGetValue(OOBFname, out obi))
                    return;
                if ((obi.flags & OOBInfo.OBI_ACTIVE) == 0)
                    // This case arises during state transfer of the initial view because the processes being
                    // initialized won't yet belong to the group and hence won't have OBI_ACTIVE set yet
                    return;
                obi.flags &= ~OOBInfo.OBI_ACTIVE;
                obi.currentlySendingTo = null;
                --OOBInUse;
            }
            if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                Isis.WriteLine("*** OOBDone for " + OOBFname + ", new OOBInUse=" + OOBInUse);
            OOBCheckQ();
        }

        private void doReportChunkStatus(string OOBFname, int nc)
        {
            OOBInfo obi;
            using (new LockAndElevate(OOBMapLock))
                if (OOBMap.TryGetValue(OOBFname, out obi))
                    if (obi.OOBStatus != null && obi.OOBStatus.Length == nc)
                    {
                        doReply(obi.OOBStatus);
                        return;
                    }
            if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                Isis.WriteLine("WARNING: doReportChunkStatus can't find OOBMap (or found it but didn't find OOBStatus) for " + OOBFname);
            doReply(new bool[nc]);
        }

        internal class AsyncReadInfo
        {
            internal byte[] buffer;
            internal MemoryMappedViewAccessor mmva;
            internal Semaphore DoneSema;
            internal OOBInfo obi;
            internal int hashcode;
            internal static int cnt;
            internal int uid = ++cnt;

            internal AsyncReadInfo(OOBInfo obi, byte[] buffer, MemoryMappedViewAccessor mmva, Semaphore DoneSema)
            {
                this.obi = obi;
                this.buffer = buffer;
                this.mmva = mmva;
                this.DoneSema = DoneSema;
                hashcode = obi.OOBFname.GetHashCode();
            }
        }

        private void doOOBPrepare(string mcaddr, string OOBFname, Address gaddr, int nchunks, long len, List<Address> where)
        {
            int chunksRcvd = 0, allRcvd = 0;
            OOBInfo obi;
            using (new LockAndElevate(OOBMapLock))
            {
                if (!OOBMap.TryGetValue(OOBFname, out obi))
                    OOBMap.Add(OOBFname, obi = new OOBInfo(OOBFname, false, null, where, gaddr));
                if (obi.OOBStatus == null)
                    obi.OOBStatus = new bool[nchunks];
            }
            if (obi.mmf == null)
            {
                try
                {
                    obi.mmf = MemoryMappedFile.CreateNew(OOBFname, len);
                }
                catch (IOException)
                {
                    if (Isis.ISIS_OOBCLONEONSAMEHOST)
                    {
                        string fname = OOBFname + "-" + Isis.ISISMEMBERS.theView.GetMyRank();
                        obi.mmf = MemoryMappedFile.CreateNew(fname, len);
                        if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                            Isis.WriteLine("WARNING: Created a new mapped file using name " + fname + " (did you run two Isis processes on the same machine?)");
                    }
                    else
                    {
                        obi.mmf = MemoryMappedFile.CreateOrOpen(OOBFname, len);
                        for (int b = 0; b < nchunks; b++)
                            obi.OOBStatus[b] = true;
                    }
                }
                if (obi.mmf == null)
                    throw new IsisException("Unable to map file " + OOBFname);
                obi.mmva = obi.mmf.CreateViewAccessor();
            }
            CheckReceived(nchunks, obi, ref chunksRcvd, ref allRcvd);
            doReply(obi.OOBStatus);
            obi.flags |= OOBInfo.OBI_RCVING;
            Thread t = new Thread(delegate()
            {
                IPAddress physIP = Isis.LastIPv4(mcaddr);
                EndPoint remoteEP = new IPEndPoint(physIP, Isis.ISIS_GROUPPORT);
                Socket rcvSock = MCMDSocket.MCMDSockSetup(physIP);
                // Limits the thread to posting 250 asynchronous receives at a time (note that 250*60K = 15Mb; we can't use the full 64Kb because of Isis Msg overheads)
                Semaphore DoneSema = new Semaphore(LOWATERMARK, int.MaxValue);
                try
                {
                    if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                        Isis.WriteLine("doOOBPrepare: chunksRcvd=" + chunksRcvd + ", nchunks=" + nchunks);
                    while (chunksRcvd < nchunks && (IsisSystem.IsisActive || IsisSystem.IsisRestarting))
                    {
                        int lim = Math.Min(CHUNKSPERTRY, nchunks - chunksRcvd);
                        for (int n = 0; n < lim; n++)
                        {
                            byte[] buffer;
                            using (new LockAndElevate(OOBMapLock))
                                if (RecycledBuffers.Count() == 0)
                                    buffer = new byte[Isis.ISIS_MAXMSGLEN];
                                else
                                {
                                    buffer = RecycledBuffers[0];
                                    RecycledBuffers.Remove(buffer);
                                }
                            AsyncReadInfo ari = new AsyncReadInfo(obi, buffer, obi.mmva, DoneSema);
                            SocketAsyncEventArgs ae = new SocketAsyncEventArgs();
                            ae.SetBuffer(buffer, 0, (int)Isis.ISIS_MAXMSGLEN);
                            ae.RemoteEndPoint = remoteEP;
                            ae.UserToken = ari; ae.SetBuffer(buffer, 0, buffer.Length);
                            ae.Completed += AsyncRcvDone;
                            if (!rcvSock.ReceiveFromAsync(ae))
                                AsyncRcvDone(null, ae);
                        }
                        int nrcvd = 0, oldchunksRcvd = chunksRcvd;
                        bool justChecked = false;
                        for (int n = 0; n < lim && chunksRcvd < nchunks; n++)
                        {
                            obi.flags |= OOBInfo.OBI_RCVWAIT;
                            justChecked = false;
                            DoneSema.WaitOne();
                            // The logic that follow is a bit tortured.  The issue is this: if packets are dropped and retransmitted,
                            // we should break out of this loop "early" once all needed chunks are received.  But if an OOB object is really
                            // big, the check that all were received could be slow, so the code that follows avoids calling CheckReceived
                            // chunk by chunk unless the number received could actually push us over the limit
                            ++nrcvd;
                            if (chunksRcvd + nrcvd >= nchunks)
                            {
                                CheckReceived(nchunks, obi, ref chunksRcvd, ref allRcvd);
                                nrcvd -= chunksRcvd - oldchunksRcvd;
                                oldchunksRcvd = chunksRcvd;
                                justChecked = true;
                            }
                            obi.flags &= ~OOBInfo.OBI_RCVWAIT;
                        }
                        if (!justChecked)
                            CheckReceived(nchunks, obi, ref chunksRcvd, ref allRcvd);
                    }
                    rcvSock.Close();
                    obi.flags &= ~OOBInfo.OBI_RCVING;
                    if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                        Isis.WriteLine("doOOBPrepare: receive logic successful, toggling mmf-ready semaphore");
                    obi.mmfReady.Release(1);
                    if (theNotifier != null)
                        theNotifier(obi.OOBFname, obi.mmf);
                }
                catch (SocketException e)
                {
                    Isis.WriteLine("doOOBPrepare: Receive thread threw a socket exception with inner code: " + e.ErrorCode);
                }
            });
            t.Name = "doOOBPrepare: Receive thread";
            t.Start();
        }

        private static void CheckReceived(int nchunks, OOBInfo obi, ref int chunksRcvd, ref int allRcvd)
        {
            while (allRcvd < nchunks && obi.OOBStatus[allRcvd])
            {
                ++chunksRcvd;
                ++allRcvd;
            }
            chunksRcvd = allRcvd;
            for (int n = allRcvd; n < nchunks; n++)
                if (obi.OOBStatus[n])
                    ++chunksRcvd;
        }

        private void AsyncRcvDone(object sender, SocketAsyncEventArgs ae)
        {
            AsyncReadInfo ari = (AsyncReadInfo)ae.UserToken;
            byte[] buffer = ari.buffer;
            ae.SetBuffer(null, 0, 0);
            Group g = Group.doLookup(ari.obi.gaddr);
            if (g != null && g.myAes != null)
                buffer = g.decipherBuf(buffer);
            bool success = Msg.InvokeFromBArray(buffer, (Action<int, long, byte[]>)delegate(int hashcode, long offset, byte[] payload)
            {
                if (hashcode != ari.hashcode)
                {
                    if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                        Isis.WriteLine("WARNING: discard a packet in AsyncRcvDone (hashcode incorrect)");
                    return;
                }
                ari.mmva.WriteArray<byte>(offset, payload, 0, payload.Length);
                if (ari.obi.OOBStatus == null)
                    throw new IsisException("OOBStatus null in AsyncRcvDone");
                int cn = (int)(offset / Isis.ISIS_OOBCHUNKSIZE);
                ari.obi.OOBStatus[cn] = true;
                ari.DoneSema.Release(1);
            });
            if (!success && (IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                Isis.WriteLine("WARNING: discard a packet in AsyncRcvDone (msg corruption)");
            using (new LockAndElevate(OOBMapLock))
                RecycledBuffers.Add(buffer);
        }

        private static long OOBGetLength(MemoryMappedViewAccessor mmva)
        {
            return mmva.Capacity;
        }

        private static byte[] OOBGetChunk(MemoryMappedViewAccessor mmva, int cn)
        {
            byte[] payload = new byte[(int)Isis.ISIS_OOBCHUNKSIZE];
            int nb = mmva.ReadArray<byte>((long)(cn * Isis.ISIS_OOBCHUNKSIZE), payload, 0, (int)Isis.ISIS_OOBCHUNKSIZE);
            if (nb < (int)Isis.ISIS_OOBCHUNKSIZE)
                Isis.ArrayResize<byte>(ref payload, nb);
            return payload;
        }

        private static long OOBGetOffset(int cn)
        {
            return (long)(cn * Isis.ISIS_OOBCHUNKSIZE);
        }

        private static void doOOBGotChunk(string OOBFname, long offset, byte[] data)
        {
            OOBInfo obi;
            using (new LockAndElevate(OOBMapLock))
                if (OOBMap.TryGetValue(OOBFname, out obi))
                    obi.mmva.WriteArray<byte>(offset, data, 0, data.Length);
        }

        /// <summary>
        /// Delete an out of band object completely
        /// </summary>
        /// <param name="OOBFname"></param>
        public void OOBDelete(string OOBFname)
        {
            if (this != Isis.ISISMEMBERS)
            {
                Isis.ISISMEMBERS.OOBDelete(OOBFname);
                return;
            }
            OOBInfo obi;
            using (new LockAndElevate(OOBMapLock))
                if (!OOBMap.TryGetValue(OOBFname, out obi))
                    return;
            obi.flags |= OOBInfo.OBI_FETCHWAIT;
            obi.mmfReady.WaitOne();
            obi.mmfReady.Release(1);
            obi.flags &= ~OOBInfo.OBI_FETCHWAIT;
            OOBReReplicate(OOBFname, new List<Address>());
        }

        /// <summary>
        /// Find the MemoryMappedFile object associated with this OOBFname.  Blocks until file is fully transferred
        /// </summary>
        /// <returns></returns>
        /// <param name="OOBFname"></param>
        public MemoryMappedFile OOBFetch(string OOBFname)
        {
            if (this != Isis.ISISMEMBERS)
                return Isis.ISISMEMBERS.OOBFetch(OOBFname);
            OOBInfo obi = null;
            using (new LockAndElevate(OOBMapLock))
                if (!OOBMap.TryGetValue(OOBFname, out obi))
                    OOBMap.Add(OOBFname, obi = new OOBInfo(OOBFname));
            if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                Isis.WriteLine("OOBFetch: checking to see if " + OOBFname + " is ready to access");
            obi.flags |= OOBInfo.OBI_FETCHWAIT;
            obi.mmfReady.WaitOne();
            obi.mmfReady.Release(1);
            obi.flags &= ~OOBInfo.OBI_FETCHWAIT;
            if ((obi.flags & OOBInfo.OBI_MASTER) == 0)
            {
                if (obi.OOBStatus == null)
                    throw new IsisException("OOBFetch: fetchwait finished yet OOBStatus=null");
                foreach (bool b in obi.OOBStatus)
                    if (!b)
                        throw new IsisException("OOBFetch synchronization error: not all blocks have transferred");
            }
            if ((IsisSystem.Debug & IsisSystem.OOBXFERS) != 0)
                Isis.WriteLine("OOBFetch: " + OOBFname + " is ready to access");
            return obi.mmf;
        }

        /// <summary>
        /// Information record returned by OOBGetRep
        /// </summary>
        [AutoMarshalled]
        public class OOBRepInfo
        {
            /// <summary>
            /// Mapped file name
            /// </summary>
            public string OOBFname;
            /// <summary>
            /// Group with which file is associated
            /// </summary>
            public Address gaddr;
            /// <summary>
            /// Locations where it can be found, in current group view
            /// </summary>
            public List<Address> OOBReplicas;

            /// <exclude></exclude>
            public OOBRepInfo()
            {
                /* Needed by Isis AutoMarshalled */
            }

            /// <exclude></exclude>
            public OOBRepInfo(string OOBFname, Address gaddr, List<Address> OOBReplicas)
            {
                this.OOBFname = OOBFname;
                this.gaddr = gaddr;
                this.OOBReplicas = OOBReplicas;
            }
        }

        /// <summary>
        /// Used when a new leader is taking over; returns the current replication pattern for the mapped objects
        /// </summary>
        /// <returns></returns>
        public List<OOBRepInfo> OOBGetRepInfo()
        {
            if (this != Isis.ISISMEMBERS)
                return Isis.ISISMEMBERS.OOBGetRepInfo();
            View theView;
            using (new LockAndElevate(ViewLock))
                theView = this.theView;
            if (theView == null)
                return null;
            List<OOBRepInfo> theRes = new List<OOBRepInfo>();
            using (new LockAndElevate(OOBMapLock))
                foreach (var oobi in OOBMap)
                    theRes.Add(new OOBRepInfo(oobi.Key, oobi.Value.gaddr, oobi.Value.replicas));
            return theRes;
        }

        // Called (in ISISMEMBERS) when view changes; any pending multicasts will have been delivered by now
        internal void OOBNewView(View v)
        {
            bool recheck = false;
            OOBInfo obi = null;
            if (v.leavers.Length == 0)
                return;
            using (new LockAndElevate(OOBMapLock))
                foreach (var kvp in OOBMap)
                {
                    Address oldLeader = null;
                    obi = kvp.Value;
                    int oldCount = obi.replicas.Count();
                    if (obi.replicas.Count() > 0)
                    {
                        oldLeader = obi.replicas[0];
                        List<Address> newReplicas = new List<Address>();
                        foreach (Address a in obi.replicas)
                            if (!v.leavers.Contains(a))
                                newReplicas.Add(a);
                        obi.replicas = newReplicas;
                    }
                    if (obi.replicas.Count() == 0 && oldCount != 0)
                    {
                        OOBRemoveObi(obi);
                        continue;
                    }
                    if ((obi.flags & OOBInfo.OBI_ACTIVE) != 0 && v.leavers.Contains(oldLeader) && obi.replicas[0].isMyAddress())
                        recheck = true;
                }
            if (recheck)
                OOBCheckQ(obi);
        }

        /// <summary>
        /// Callback used on completion of an OOBBeginFetch
        /// </summary>
        /// <param name="OOBFname"></param>
        /// <param name="mmf"></param>
        public delegate void OOBAsyncCallback(string OOBFname, MemoryMappedFile mmf);

        /// <summary>
        /// Asynchronously find the MemoryMappedFile object for this OOBFname.  Callback occurs when the file is fully transferred
        /// </summary>
        /// <param name="acb"></param>
        /// <param name="OOBFname"></param>
        public void OOBBeginFetch(string OOBFname, OOBAsyncCallback acb)
        {
            if (this != Isis.ISISMEMBERS)
            {
                Isis.ISISMEMBERS.OOBBeginFetch(OOBFname, acb);
                return;
            }
            Thread t = new Thread(delegate()
            {
                for (int retry = 0; retry < 5; retry++)
                {
                    OOBInfo obi;
                    using (new LockAndElevate(OOBMapLock))
                        if (OOBMap.TryGetValue(OOBFname, out obi))
                        {
                            acb(OOBFname, obi.mmf);
                            return;
                        }
                    Thread.Sleep(10000);
                }
                throw new IsisException("OOBBeginFetch didn't find an entry for < " + OOBFname + ">");
            });
            t.Name = "OOBBeginFetch callback thread for <" + OOBFname + ">";
            t.Start();
        }

        private static bool disposed;

        /// <summary>
        /// Because Group uses some unmanaged resources, C# requires an explicit "Dispose" API
        /// </summary>
        public void Dispose()
        {
            Dispose(true);
        }

        private void Dispose(bool disposing)
        {
            lock (this)
                if (disposed)
                    return;
                else
                    disposed = true;
            if (!disposing)
                return;
            if (Wedged != null)
            {
#if !__MonoCS__
                Wedged.Dispose();
#endif // !__MonoCS__
                Wedged = null;
            }
            if (myChkptStream != null)
            {
                myChkptStream.Dispose();
                myChkptStream = null;
            }
        }
    }

    /// <summary>
    /// The Address object represents a process or group within the Isis system
    /// </summary>
    /// <remarks>
    /// The Address object represents a process or group within the Isis system
    /// Encoded within it are the IP address of the node on which the process lives, its process-id
    /// and the port numbers used to send to it.  However, although this information is printed by the
    /// pretty-print routines, it is not normally needed by Isis users and is not accessible other than
    /// in print form.
    /// </remarks>
    public class Address : IComparable<Address>, IEquatable<Address>
    {
        internal IPAddress home;                      // Node on which this process is running
        internal int pid;                             // Process identifier for a process; 0 for a group
        internal Int32 p2pPort;                       // Personal port number for point-to-point messages
        internal Int32 ackPort;                       // Personal port number for acks
        internal int cachedHashCode;                  // To speed up the computation of the hashcode

        internal static int GroupNameHash(String gname)
        {
            if (gname == null)
                throw new GroupNotFound("ISIS can't join anonymous groups\r\n" + IsisSystem.GetState());
            if (gname.Equals("ORACLE"))
                return 0;
            int the_hash = 0;
            for (int i = 0; i < gname.Length; i++)
            {
                unchecked { the_hash = the_hash * 37 + gname[i]; }
            }
            if ((the_hash &= 0xFFFF) == 0) the_hash = 1;
            return the_hash % (Isis.ISIS_MCRANGE_HIGH - Isis.ISIS_MCRANGE_LOW);
        }

        /// <summary>
        /// Implements the C# equality API for Address objects
        /// </summary>
        /// <param name="p2">the address to compare against</param>
        /// <returns>true if the addresses refer to the same process or same group</returns>
        public override bool Equals(Object p2)
        {
            if (p2 == null || !p2.GetType().Equals(typeof(Address)))
                return false;
            return Equals((Address)p2);
        }

        /// <summary>
        /// Implements the C# equality API for Address objects
        /// </summary>
        /// <param name="p2">the address to compare against</param>
        /// <returns>true if the addresses refer to the same process or same group</returns>
        public bool Equals(Address p2)
        {
            if (p2 == null) return false;
            byte[] hb1 = home.GetAddressBytes(), hb2 = p2.home.GetAddressBytes();
            // For a group, pid is always 0 and we compare virtual IPMC IP address number and port numbers.  For a process, we use the host and pid numbers.
            int h1 = ((int)hb1[0] << 24) | ((int)hb1[1] << 16) | ((int)hb1[2] << 8) | (int)hb1[3];
            int h2 = ((int)hb2[0] << 24) | ((int)hb2[1] << 16) | ((int)hb2[2] << 8) | (int)hb2[3];
            if (h1 != h2 || pid != p2.pid || (pid == 0 && p2.pid == 0 && ackPort != p2.ackPort || p2pPort != p2.p2pPort)) return false;
            return true;
        }

        /// <summary>
        /// Implements the C# comparison API for Address objects
        /// </summary>
        /// <param name="b">address to compare with</param>
        /// <returns>-1, 0 or 1 on the basis of an Address sort rule</returns>
        public int CompareTo(Address b)
        {
            if (b == null)
                return -1;
            byte[] hb1 = home.GetAddressBytes(), hb2 = b.home.GetAddressBytes();
            // For a group, pid is always 0 and we compare virtual IPMC IP address number and port numbers.  For a process, we use the host and pid numbers.
            int h1 = ((int)hb1[0] << 24) | ((int)hb1[1] << 16) | ((int)hb1[2] << 8) | (int)hb1[3];
            int h2 = ((int)hb2[0] << 24) | ((int)hb2[1] << 16) | ((int)hb2[2] << 8) | (int)hb2[3];
            if (h1 != h2) return h1 - h2;
            if (pid == 0 && b.pid == 0)
            {
                if (ackPort != b.ackPort)
                    return ackPort - b.ackPort;
                if (p2pPort != b.p2pPort)
                    return p2pPort - b.p2pPort;
            }
            return pid - b.pid;
        }

        /// <summary>
        /// Tests to see if some address b is included in some list of addresses alist
        /// </summary>
        /// <param name="alist">list of addresses</param>
        /// <param name="b">process we are looking for</param>
        /// <returns>true if the list includes the designated process, else false</returns>
        public static bool Includes(Address[] alist, Address b)
        {
            foreach (Address a in alist)
                if (a.Equals(b))
                    return true;
            return false;
        }

        /// <summary>
        /// Tests to see if the target Address is the caller's address
        /// </summary>
        /// <returns>true if the Address is the address of the calling process, false if not</returns>
        public bool isMyAddress()
        {
            return Equals(Isis.my_address);
        }

        /// <summary>
        /// Tests to see if the target Address is the NULL address
        /// </summary>
        /// <returns>true if the Address is the NULL address, false if not</returns>
        public bool isNull()
        {
            return Equals(Isis.NULLADDRESS);
        }

        /// <summary>
        /// Tests to see if the caller is in some list of Addresses
        /// </summary>
        /// <param name="list">address list to scan</param>
        /// <returns>true if the list includes the caller, false if not</returns>
        public static bool IncludesMe(Address[] list)
        {
            foreach (Address a in list)
                if (a.Equals(Isis.my_address))
                    return true;
            return false;
        }

        /// <summary>
        /// Overrides GetHashCode for Address comparisons
        /// </summary>
        /// <returns>unique hashcode for the Address</returns>
        public override int GetHashCode()
        {
            if (cachedHashCode != 0)
                return cachedHashCode;
            byte[] hb = home.GetAddressBytes();
            // For a group, pid is always 0 and we compare virtual IPMC IP address number and port numbers.  For a process, we use the host and pid numbers.
            if (pid == 0)
                cachedHashCode = ackPort * 171 + p2pPort * 133 + ((int)hb[0] * 13 << 24) | ((int)hb[1] * 13 << 16) | ((int)hb[2] * 13 << 8) | (int)hb[3] * 13;
            else
                cachedHashCode = pid * 370037 + (pid & 1) + ((pid >> 2) & 1) + ((int)hb[0] * 13 << 24) | ((int)hb[1] * 13 << 16) | ((int)hb[2] * 13 << 8) | (int)hb[3] * 13;
            cachedHashCode &= 0x7FFFFFFF;
            return cachedHashCode;
        }

        internal Address(IPAddress h, int p)
        {
            home = h;
            pid = p;
        }

        /// <summary>
        /// This method is used to turn a string formatted Address back into an Isis Address object.
        /// </summary>
        /// <param name="aStr">The address in the printed outform used by Isis</param>
        /// <remarks>Used in large configurations for Master/Worker batch starts.   Requires long-form address from toString().  
        /// If the IsisSystem.Debug VERBOSEADDRS bit isn't enabled, the Address.ToString() output will be in a short-hand form that makes for
        /// concise printouts but lacks some of the required fields and hence will cause Address(targetAddress) to throw an exception. </remarks>
        public Address(string aStr)
        {
            string PID, b0, b1, b2, b3, P2PPORT;
            // Format:  "(" + pid.ToString() + ":" + b[0] + "." + b[1] + "." + b[2] + "." + b[3] + "/" + p2pPort + ":" + ackPort + ")";
            if (aStr[0] != '(' || aStr[aStr.Length - 1] != ')')
                goto fmt;
            aStr = aStr.Substring(1, aStr.Length - 2);
            if ((PID = aChunk(ref aStr, ':')) == null || (b0 = aChunk(ref aStr, '.')) == null || (b1 = aChunk(ref aStr, '.')) == null || (b2 = aChunk(ref aStr, '.')) == null || (b3 = aChunk(ref aStr, '/')) == null || (P2PPORT = aChunk(ref aStr, ':')) == null)
                goto fmt;
            try
            {
                pid = int.Parse(PID);
                home = new IPAddress(new byte[] { (byte)int.Parse(b0), (byte)int.Parse(b1), (byte)int.Parse(b2), (byte)int.Parse(b3) });
                p2pPort = int.Parse(P2PPORT);
                ackPort = int.Parse(aStr);
                cachedHashCode = 0;
                return;
            }
            catch { }
        fmt:
            throw new ArgumentException("Address format should be \"(pid:wwww.xxxx.yyyy.zzzz/pport:aport)\"");
        }

        // Eats a chunk of text up to a delimiter character, returns the text, truncates the string and leaves the remainder
        private static string aChunk(ref string aStr, char c)
        {
            string pid;
            int i = aStr.IndexOf(c);
            if (i == -1)
                return null;
            pid = aStr.Substring(0, i);
            aStr = aStr.Substring(i + 1);
            return pid;
        }

        internal const int lengthOfAddress = 16;

        internal void toBArray(byte[] b, int off)
        {
            byte[] myIP = home.GetAddressBytes();
            int myPID = pid, myP2P = p2pPort, myAP = ackPort;
            for (int i = 0; i < 4; i++)
            {
                b[off + i] = myIP[i];
                b[off + 4 + i] = (byte)(myP2P & 0xFF);
                b[off + 8 + i] = (byte)(myAP & 0xFF);
                b[off + 12 + i] = (byte)(myPID & 0xFF);
                myP2P >>= 8; myAP >>= 8; myPID >>= 8;
            }
        }

        internal Address(byte[] b, int off)
        {
            byte[] myIP = new byte[4];
            int myPID = 0, myP2P = 0, myAP = 0;
            for (int i = 0; i < 4; i++)
            {
                myIP[i] = b[off + i];
                myP2P = (myP2P << 8) | b[off + 7 - i];
                myAP = (myAP << 8) | b[off + 11 - i];
                myPID = (myPID << 8) | b[off + 15 - i];
            }
            home = new IPAddress(myIP);
            pid = myPID;
            p2pPort = myP2P;
            ackPort = myAP;
        }

        internal byte[] toBArray()
        {
            byte[] b = new byte[lengthOfAddress];
            toBArray(b, 0);
            return b;
        }

        internal static byte[] toBArray(Address[] al)
        {
            byte[] b = new byte[al.Length * lengthOfAddress];
            for (int off = 0; off < al.Length; off++)
            {
                Address a = al[off];
                if (a == null) a = Isis.NULLADDRESS;
                a.toBArray(b, off * lengthOfAddress);
            }
            return b;
        }

        internal static Address[] ToAddresses(byte[] ba, int off, int nb)
        {
            int na = nb / lengthOfAddress;
            if (na * lengthOfAddress != nb)
                return new Address[0];
            Address[] av = new Address[na];
            for (int n = 0; n < na; n++)
            {
                av[n] = new Address(ba, off); off += lengthOfAddress;
            }
            return av;
        }

        internal static bool SameAddrs(Address[] a, Address[] b)
        {
            bool same = true;
            if (a.Length != b.Length)
                return false;
            for (int i = 0; i < a.Length && same; i++)
                if (!a[i].Equals(b[i]))
                    same = false;
            if (same)
                return true;
            foreach (Address aa in a)
            {
                same = false;
                foreach (Address ba in b)
                    if (aa.Equals(ba))
                    {
                        same = true;
                        break;
                    }
                if (!same)
                    return false;
            }
            return true;
        }

        internal static bool SameNames(string[] a, string[] b)
        {
            bool same = true;
            if (a.Length != b.Length)
                return false;
            for (int i = 0; i < a.Length && same; i++)
                if (!a[i].Equals(b[i]))
                    same = false;
            if (same)
                return true;
            foreach (string astr in a)
            {
                same = false;
                foreach (string bstr in b)
                    if (astr.Equals(bstr))
                    {
                        same = true;
                        break;
                    }
                if (!same)
                    return false;
            }
            return true;
        }

        internal Address() { home = new IPAddress(0); }

        /// <summary>
        /// Pretty-print a single Address
        /// </summary>
        /// <returns>string representation of the Address</returns>
        public override string ToString()
        {
            if ((IsisSystem.Debug & IsisSystem.VERBOSEADDRS) != 0 || (pid == 0 && !isNull()))
                return ToStringVerboseFormat();
            return "(" + pid.ToString() + ")";
        }

        /// <summary>
        /// Returns true if this is a group address, false if it is a process address
        /// </summary>
        /// <returns></returns>
        public bool isGroupAddress()
        {
            return !Equals(Isis.NULLADDRESS) && pid == 0;
        }

        /// <summary>
        /// Pretty-print a single Address in the detailed format required for parsing back into an Address via new Address(str)
        /// </summary>
        /// <returns>string representation of the Address</returns>
        /// <remarks>To keep our traces short, we normally print addresses in a very short format</remarks>
        public string ToStringVerboseFormat()
        {
            byte[] b = home.GetAddressBytes();
            return "(" + pid.ToString() + ":" + b[0] + "." + b[1] + "." + b[2] + "." + b[3] + "/" + p2pPort + ":" + ackPort + ")";
        }

        /// <summary>
        /// Pretty-print a vector of Addresses
        /// </summary>
        /// <param name="a">the address vector</param>
        /// <returns>string representation of the vector</returns>
        public static string VectorToString(Address[] a)
        {
            if (a == null) return "";
            string theString = "";
            for (int i = 0; i < a.Length; i++)
                theString += a[i].ToString();
            return "[" + theString + "]";
        }

        internal static Address[] Merge(Address[] a, Address b)
        {
            return Merge(a, new Address[] { b });
        }

        internal static Address[] Merge(Address[] a, Address[] b)
        {
            if (a == null)
                a = new Address[0];
            if (b == null)
                b = new Address[0];
            Address[] Merged = new Address[a.Length + b.Length];
            int n = 0;
            for (int i = 0; i < a.Length; i++)
            {
                bool dup = false;
                for (int j = 0; j < n; j++)
                    if (Merged[j].Equals(a[i]))
                    {
                        dup = true;
                        break;
                    }
                if (!dup)
                    Merged[n++] = a[i];
            }
            for (int i = 0; i < b.Length; i++)
            {
                bool dup = false;
                for (int j = 0; j < a.Length; j++)
                    if (a[j].Equals(b[i]))
                    {
                        dup = true;
                        break;
                    }
                if (!dup)
                    Merged[n++] = b[i];
            }
            if (n != Merged.Length)
                Isis.ArrayResize<Address>(ref Merged, n);
            return Merged;
        }

        internal static Address[] Diff(Address[] a, Address[] b)
        {
            if (a == null || b == null)
                return a;
            Address[] r = new Address[a.Length];
            int n = 0;
            for (int i = 0; i < r.Length; i++)
            {
                bool fnd = false;
                for (int j = 0; j < b.Length; j++)
                    if (a[i].Equals(b[j]))
                    {
                        fnd = true;
                        break;
                    }
                if (!fnd)
                    r[n++] = a[i];
            }
            if (r.Length != n)
                Isis.ArrayResize<Address>(ref r, n);
            return r;
        }
    }

    /*
     *  This slightly messy logic allows the View members structure to switch back and forth between being indexed for fast
     *  search (needed with large groups) and acting like a totally normal vector of Address objects (for small ones).  Users
     *  of the v.members property shouldn't be aware of the underlying logic because it mimics a vector API.
     */
    /// <exclude></exclude>
    public class indexedMVector
    {
        private Address[] _members = new Address[0];
        private Dictionary<Address, int> index = null;
        /// <exclude></exclude>
        public Address[] members
        {
            get
            {
                return _members;
            }
            set
            {
                _members = value;
                if (_members.Length > Isis.ISIS_INDEXMEMBERS)
                {
                    index = new Dictionary<Address, int>(1000);
                    for (int i = 0; i < _members.Length; i++)
                        if (_members[i] != null && !_members[i].isNull())
                            index.Add(_members[i], i);
                }

            }
        }
        /// <exclude></exclude>
        public Address this[int i]
        {
            get
            {
                return _members[i];
            }
            set
            {
                if (index != null)
                {
                    if (_members[i] != null && !_members[i].isNull())
                        index.Remove(_members[i]);
                    if (value == null && !value.isNull())
                        index.Add(value, i);
                }
                _members[i] = value;
            }
        }
        /// <exclude></exclude>
        public int IndexOf(Address a)
        {
            if (index == null || a.isNull())
            {
                for (int i = 0; i < members.Length; i++)
                    if (members[i].Equals(a))
                        return i;
                return -1;
            }
            int where;
            if (index.TryGetValue(a, out where))
                return where;
            return -1;
        }
    }

    /// <summary>
    /// The View structure for an Isis Group.  This is a central abstraction for the system with many useful
    /// guarantees.  Read about the <it>virtual synchrony model</it> to learn more.
    /// </summary>
    /// <remarks>
    /// The core abstraction in the Isis platform is the virtual synchrony model, which provides a unique, globally observed ordering on
    /// messages and membership changes, which are reported through new "View" events.  The view structure encodes all the information
    /// needed to coordinate a set of processes.  Views are reported via callback from a per-group thread created by Isis.
    /// 
    /// All members of the view see the same view, with members in the same order.  That order is from oldest (longest resident in the group)
    /// to youngest, with ties broken by the order in which the join requests reached the core Isis system.  The list of joiners indicates the
    /// members that just joined, as a convenience to the called view monitor.  The list of leavers, similarly, lists members that belonged to the
    /// prior view but are no longer in the group (e.g. they failed, or voluntarily departed).
    /// 
    /// The view is uniquely numbered by a sequential id value.  Once view K has been reported, no further 
    /// multicasts will be delivered in view K-1, and all operational members of view K-1 has received the identical messages.
    /// </remarks>
    public class View
    {
        /// <summary>
        /// Isis-assigned Address representing this group
        /// </summary>
        public Address gaddr;
        /// <summary>
        /// Group name, if known
        /// </summary>
        public string gname = "unknown";
        private indexedMVector _members = new indexedMVector();
        /// <summary>
        /// List of the current members of this group
        /// </summary>
        public Address[] members
        {
            get
            {
                return _members.members;
            }
            set
            {
                _members.members = value;
            }
        }
        /// <summary>
        /// List of the processes that joined this group since the last view was reported
        /// </summary>
        public Address[] joiners = new Address[0];
        /// <summary>
        /// List of processes that departed this group since the last view was reported
        /// </summary>
        public Address[] leavers = new Address[0];
        /// <summary>
        /// If members are joining, this offset is an optional long offset value that "indexes" into the
        /// state to be transferred.  The checkpoint creation routine can use this value to omit state that
        /// the joining process already has obtained (somehow)
        /// </summary>
        public long offset;
        internal List<ChkptMaker> theChkptMakers;
        internal Address currentOracleRep = Isis.NULLADDRESS;
        internal bool isFinal = false;
        internal bool isLarge = false;
        internal int leaderId;
        internal bool[] hasFailed = new bool[0];
        internal bool hasFailedHasChanged;
        internal int[] NextIncomingMsgID;
        internal int[] StableTo;
        internal int[] LastMsg;
        internal int[] myVT = new int[0];
        internal int myClock = 0;
        /// <summary>
        /// View id for the group
        /// </summary>
        public int viewid;
        internal int nextMsgid;   // Used to initialize the seqn of a joining member in a large group.  Group.nextMsgid is the "real" nextMsgid counter.

        /// <summary>
        /// View equality comparison method
        /// </summary>
        /// <param name="v">the view to compare against</param>
        /// <returns>true if the two views have the same view-id, else false</returns>
        public bool Equals(View v)
        {
            return gaddr.Equals(v.gaddr) && viewid == v.viewid;
        }

        /// <summary>
        /// Hashcode for this view
        /// </summary>
        /// <returns>hash code for a group view</returns>
        public override int GetHashCode()
        {
            return gaddr.GetHashCode() | viewid.GetHashCode();
        }

        /// <summary>
        /// A public method for pretty-printing a group view
        /// </summary>
        /// <returns>string representation of a group view</returns>
        public override string ToString()
        {

            string hf = "*", mid, stableto = "";
            if (!isLarge)
                mid = " <" + viewid + ":" + NextIncomingMsgID[0] + "> ";
            else
                mid = "";
            for (int i = 1; i < hasFailed.Length + 1; i++)
            {
                hf += (hasFailed[i - 1]) ? "+" : "-";
                if (i < NextIncomingMsgID.Length)
                {
                    mid += (!isLarge ? (viewid + ":") : "") + NextIncomingMsgID[i] + " ";
                    if (!isLarge)
                        stableto += StableTo[i] + " ";
                }
            }
            if (stableto.Length > 0)
                stableto = ", StableTo={ ** " + stableto + "}";
            string rv = ViewSummary() + ", hasFailed=[" + hf + "], nextIncomingMSGID={ " + mid + "}" + stableto + ", joining={ " + Address.VectorToString(joiners) + "}, leaving={ " + Address.VectorToString(leavers) + "}";
            if (!isLarge)
                rv += ", IamLeader = " + IAmLeader();
            return rv;
        }

        internal string ViewSummary()
        {
            return "View[gname=<" + gname + ">, gaddr=" + gaddr + ", viewid=" + viewid + ";  " + members.Length + " members={ " + Address.VectorToString(members) + "}";
        }

        /// <summary>
        /// Returns the current view id for this view
        /// </summary>
        /// <returns>id of the current view</returns>
        public int GetViewid() { return viewid; }

        /// <summary>
        /// Returns the rank of the calling process in the view, or -1 if the caller is not a member of the group
        /// </summary>
        /// <returns>rank of the caller</returns>
        public int GetMyRank() { return GetRawRankOf(Isis.my_address); }


        /// <summary>
        /// Obtains the current rank of a specified process within the current view, but returns -1 if the process has failed.
        /// </summary>
        /// <remarks>
        /// GetRankOf returns the value that <see cref="GetRawRankOf"/> would have returned except that if the process in question has failed, it returns -1.  Thus, calls to
        /// GetRankOf on a view {x,y,z} where y has failed would return 0 for x, -1 for y, and 1 for z.  Returns -1 if the
        /// member is not in the group at all, hence also returns -1 for w in this example.</remarks>
        /// <param name="a">address of the member</param>
        /// <returns>rank of the designated member</returns>
        public int GetRankOf(Address a)
        {
            int which = _members.IndexOf(a);
            if (which != -1 && !hasFailed[which])
                return which;
            return -1;
        }

        /// <summary>
        /// Obtains the rank of a specified process within the current view, returning an index into the current view even if the process has failed.  
        /// </summary>
        /// <remarks>
        /// Rank is defined from
        /// 0 (oldest) to n-1 (youngest) for a group with n members.  Ties are broken by using the original order in which Isis was
        /// told to add a set of members: if the system was asked to add x, y and z all at once, x will rank lower than y, etc.
        /// Returns -1 if the specified process is not a member of the view.</remarks>
        /// <param name="a">address of the member</param>
        /// <returns>rank of that member</returns>
        public int GetRawRankOf(Address a)
        {
            return _members.IndexOf(a);
        }

        /// <summary>
        /// True if the caller is the leader (oldest, lowest-ranked) member of this view.
        /// </summary>
        /// <remarks>
        /// This member is often used as the leader in tasks where a single unambiguous leader is needed.
        /// </remarks>
        /// <returns>true if this member is the leader, false if not</returns>
        public bool IAmLeader()
        {
            int i;
            for (i = 0; i < members.Length; i++)
                if (!hasFailed[i])
                    break;
            if (i == members.Length)
                return false;
            return members[i].isMyAddress();
        }

        internal bool IAmRank0()
        {
            return members.Length > 0 && members[0].isMyAddress();
        }

        /// <summary>
        /// Obtains a vector of Addresses[] giving the current membership of this group
        /// </summary>
        /// <returns></returns>
        public Address[] GetMembers() { return members; }

        /// <summary>
        /// Returns the number of members in the current view of this group
        /// </summary>
        /// <returns></returns>
        public int GetSize() { return members.Length; }

        /// <summary>
        /// Obtains the current view for the group corresponding to Address gaddr.  Returns null if the caller is not a member of the group.
        /// </summary>
        /// <param name="gaddr"></param>
        /// <returns></returns>
        public static View Lookup(Address gaddr) { Group g = Group.doLookup(gaddr); if (g == null) return null; return g.theView; }

        internal View(String gn, Address ga, Address[] m, int vid, bool lg)
        {
            gname = gn;
            gaddr = ga;
            members = (Address[])m.Clone();
            myVT = new int[members.Length];
            viewid = vid;
            isLarge = lg;
            if (!isLarge)
            {
                NextIncomingMsgID = new int[members.Length + 1];
                StableTo = new int[members.Length + 1];
            }
            else
            {
                NextIncomingMsgID = new int[2];
                StableTo = new int[2];
            }
            LastMsg = new int[members.Length + 1];
            for (int i = 0; i < members.Length; i++)
                LastMsg[i] = -1;
            hasFailed = new bool[members.Length];
            hasFailedHasChanged = false;
        }

        internal View(int len, bool isLarge)
        {
            gaddr = Isis.NULLADDRESS;
            members = new Address[len];
            myVT = new int[len];
            if (!isLarge)
            {
                NextIncomingMsgID = new int[len + 1];
                StableTo = new int[len + 1];
            }
            else
            {
                NextIncomingMsgID = new int[2];
                StableTo = new int[2];
            }
            LastMsg = new int[len];
            for (int i = 0; i < len; i++)
                LastMsg[i] = -1;
            hasFailed = new bool[len];
            hasFailedHasChanged = false;
        }

        /// <summary>
        /// Notes that the r'th ranked member of the current view has failed.  For internal use only
        /// </summary>
        /// <param name="r">rank of the member that failed</param>
        public void noteFailed(int r)
        {
            if (r < 0 || r >= hasFailed.Length)
                return;
            if (!hasFailed[r])
            {
                hasFailed[r] = true;
                hasFailedHasChanged = true;
            }
            AwaitReplies.doNoteFailure(members[r]);
        }

        /// <summary>
        /// Notes that the r'th ranked member of group g has failed.  Intended primarily for use in the Isis system.
        /// </summary>
        /// <param name="g">group in which the failure was observed.</param>
        /// <param name="a">member that failed</param>
        public static void noteFailed(Group g, Address a)
        {
            // A bit cautious due to possible races against changing views
            View v;
            using (new LockAndElevate(g.ViewLock))
                v = g.theView;
            int r = v.GetRankOf(a);
            v.noteFailed(r);
            if (!g.isTrackingProxy)
            {
                Group tpg = Group.TrackingProxyLookup(g.gaddr);
                if (tpg == null)
                    return;
                using (new LockAndElevate(tpg.ViewLock))
                    v = tpg.theView;
                if (v != null && (r = tpg.theView.GetRankOf(a)) != -1)
                    v.noteFailed(r);
            }
        }

        internal int nLive()
        {
            int n = members.Length;
            foreach (bool b in hasFailed)
                if (b)
                    --n;
            return Math.Max(0, n);
        }

        internal int nFailed()
        {
            int n = 0;
            foreach (bool b in hasFailed)
                if (b)
                    ++n;
            return Math.Min(members.Length, n);
        }

        internal Address[] GetFailedMembers()
        {
            Address[] list = new Address[nFailed()];
            int idx = 0;
            for (int i = 0; i < members.Length; i++)
                if (hasFailed[i])
                    list[idx++] = members[i];
            return list;
        }

        internal Address[] GetLiveMembers()
        {
            Address[] list = new Address[nLive()];
            int idx = 0;
            for (int i = 0; i < members.Length; i++)
                if (!hasFailed[i])
                    list[idx++] = members[i];
            return list;
        }

        /// <summary>
        /// Converts View objects to an external form for transmission.  Not normally needed by end users
        /// </summary>
        /// <returns></returns>
        public byte[] toBArray()
        {
            byte[] ba = Msg.toBArray(Isis.LeaderId, gname, gaddr, members, hasFailed, viewid, NextIncomingMsgID, isLarge);
            return ba;
        }

        /// <summary>
        /// Constructor for new View objects received as byte arrays.  Not normally needed by end users.
        /// </summary>
        /// <param name="ba"></param>
        public View(byte[] ba)
        {
            object[] obs = Msg.BArrayToObjects(ba, typeof(int), typeof(String), typeof(Address), typeof(Address[]), typeof(bool[]), typeof(int), typeof(int[]), typeof(bool));
            int idx = 0;
            leaderId = (int)obs[idx++];
            gname = (String)obs[idx++];
            gaddr = (Address)obs[idx++];
            members = (Address[])obs[idx++];
            myVT = new int[members.Length];
            hasFailed = (bool[])obs[idx++];
            viewid = (int)obs[idx++];
            NextIncomingMsgID = (int[])obs[idx++];
            StableTo = new int[NextIncomingMsgID.Length];
            isLarge = (bool)obs[idx++];
        }
    }

    internal class AwaitReplies
    {
        private static LockObject UIDLock = new LockObject("UIDLock");
        private static int UID;

        internal class ReplyInfo
        {
            internal View rdvView;
            internal Address gaddr;
            internal string gname;
            internal bool enciphered;
            internal int uid;
            internal int rdvVid;
            internal int rdvMid;
            internal bool queryByOracle;
            internal List<byte[]> rdvReplies;
            internal bool[] gotReply;
            internal int replies_wanted;
            internal int replies_received = 0;
            internal int reply_waitid;
            internal string theReason = "<AbortReply>";
            internal int delay;
            internal int tid;
            internal ILock ilock;
            internal Group.VHCallBack vhcb;
            internal static int riCNTR;
            internal int ricntr = ++riCNTR;
            internal LockObject Lock = new LockObject("RdvInfo.Lock");

            internal ReplyInfo(Group g, int vid, int mid, bool sentByOracle, bool crypto, View v, List<Address> dests, int nr, int d, Group.VHCallBack vcb)
            {
                using (new LockAndElevate(UIDLock))
                    uid = UID++;
                if ((rdvView = v) != null)
                {
                    gotReply = new bool[v.members.Length];
                    if (dests != null)
                    {
                        for (int r = 0; r != v.members.Length; r++)
                            gotReply[r] = true;
                        int wc = 0;
                        foreach (var waitFor in dests)
                        {
                            int rank;
                            if ((rank = v.GetRankOf(waitFor)) != -1 && gotReply[rank])
                            {
                                ++wc;
                                gotReply[rank] = false;
                            }
                        }
                        nr = Math.Min(nr, wc);
                    }
                }
                else
                    gotReply = new bool[nr];
                if (g != null)
                {
                    gaddr = g.gaddr;
                    gname = g.gname;
                }
                else
                {
                    gaddr = Isis.NULLADDRESS;
                    gname = "<null>";
                }
                rdvVid = vid;
                rdvMid = mid;
                queryByOracle = sentByOracle;
                enciphered = crypto;
                replies_wanted = nr;
                delay = d;
                rdvReplies = new List<byte[]>();
                if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                    Isis.WriteLine("Allocate RdvInfo[" + ricntr + "]: " + rdvVid + ":" + rdvMid + " in group " + gaddr + ".... ri.wanted=" + replies_wanted);
                vhcb = vcb;
                g.doRegisterViewCB(vcb);
            }

            // Caller has a lock on ri
            internal void UpdateRepliesWanted(int from)
            {
                UpdateRepliesWanted(false, from);
            }

            // Caller has a lock on ri
            internal void UpdateRepliesWanted(bool isNullreply, int from)
            {
                if (gotReply[from])
                    return;
                gotReply[from] = true;
                if (isNullreply)
                {
                    int cnt = 0;
                    foreach (bool b in gotReply)
                        if (!b)
                            ++cnt;
                    replies_wanted = Math.Min(cnt, replies_wanted);
                }
                else
                    --replies_wanted;
                if (replies_wanted == 0 && ilock != null)
                {
                    ilock.BarrierRelease(1);
                    ilock = null;
                }
                if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                    Isis.WriteLine("UpdateRepliesWanted[" + ricntr + ": from " + from + "]: " + rdvVid + ":" + rdvMid + " in group " + gaddr + ".... --ri.wanted=" + replies_wanted);
            }
        }

        static List<ReplyInfo> RendezVous = new List<ReplyInfo>();
        static LockObject RendezVousLock = new LockObject("RendezVousLock");

        internal static string GetState()
        {
            string s = "AwaitReplies State:\r\n";
            using (new LockAndElevate(RendezVousLock))
                foreach (ReplyInfo ri in RendezVous)
                    using (new LockAndElevate(ri.Lock))
                    {
                        string from = "", needed = "";
                        if (ri.rdvView.members.Length == ri.gotReply.Length)
                            for (int i = 0; i < ri.gotReply.Length; i++)
                                if (ri.gotReply[i])
                                    from += ri.rdvView.members[i];
                                else
                                    needed += ri.rdvView.members[i];
                        s += "  Rendezvous[" + ri.uid + "]: <" + ri.gname + ">, using lock " + ILock.PLock(ri.ilock) + ", " + ri.rdvVid + ":" + ri.rdvMid + ", tid = " + ri.tid + " Need " + ri.replies_wanted +
                            " additional replies (got " + ri.replies_received + " replies from <" + from + ">";
                        if (ri.replies_wanted > 0)
                            s += ", waiting for replies from <" + needed + ">)";
                        s += ", reply wait-id " + ri.reply_waitid + "\r\n";
                    }
            return s;
        }

        private static int maxRdvId;

        // Used in P2P queries
        internal static ReplyInfo registerP2PWait(Group who, int vid, int msgid, bool expectCiphered, Address dest, int delay)
        {
            return registerWait(who, vid, msgid, false, expectCiphered, 1, new List<Address>() { dest }, delay);
        }


        internal static ReplyInfo registerGroupWait(Group who, int vid, int msgid, List<Address> destList, bool sentByOracle, bool expectCiphered, int nreplies, int delay)
        {
            return registerWait(who, vid, msgid, sentByOracle, expectCiphered, nreplies, destList, delay);
        }

        internal static ReplyInfo registerWait(Group who, int vid, int msgid, bool sentByOracle, bool expectCiphered, int nreplies, List<Address> dests, int delay)
        {
            ReplyInfo ri = null;
            if (nreplies == 0)
                return null;
            View theView = null;
            if (who != null)
                using (new LockAndElevate(who.ViewLock))
                    theView = who.theView;
            ri = new ReplyInfo(who, vid, msgid, sentByOracle, expectCiphered, theView, dests, nreplies, delay, new Group.VHCallBack(false, (ViewHandler)delegate(View v)
            {
                // This code will be executed on new-view events for the group; needed in case someone voluntarily leaves a group while a request is pending
                if (ri == null)
                    throw new IsisException("View Callback in registerWait: ri unexpectedly null");
                using (new LockAndElevate(ri.Lock))
                {
                    if (!v.gaddr.Equals(ri.gaddr))
                        Isis.WriteLine("WARNING: View handler callback to RdvInfo.registerWait but for wrong group!");
                    else foreach (var a in v.leavers)
                        {
                            int i = ri.rdvView.GetRawRankOf(a);
                            if (i != -1 && ri.gotReply[i] == false)
                            {
                                ri.UpdateRepliesWanted(i);
                                if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                                    Isis.WriteLine("[0]: ri.wanted=" + ri.replies_wanted + ", crashed: " + ri.rdvView.members[i]);
                            }
                        }
                }
            }));

            using (new LockAndElevate(RendezVousLock))
            {
                using (new LockAndElevate(ri.Lock))
                {
                    if (dests == null)
                    {
                        for (int m = 0; m < who.theView.members.Length; m++)
                            if (who.theView.hasFailed[m])
                            {
                                int n = ri.rdvView.GetRawRankOf(who.theView.members[m]);
                                if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                                    Isis.WriteLine("[5]: ri.wanted=" + ri.replies_wanted + ", already dead " + ri.rdvView.members[n]);
                                ri.UpdateRepliesWanted(true, n);
                            }
                    }
                    else
                    {
                        foreach (var dest in dests)
                        {
                            int m = theView.GetRawRankOf(dest);
                            if (m == -1 || theView.hasFailed[m])
                            {
                                int n = ri.rdvView.GetRawRankOf(who.theView.members[m]);
                                if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                                    Isis.WriteLine("[5]: ri.wanted=" + ri.replies_wanted + ", already dead " + ri.rdvView.members[n]);
                                ri.UpdateRepliesWanted(true, n);
                            }
                        }
                    }
                    if (ri.replies_wanted > 0)
                    {
                        bool[] Ids = new bool[maxRdvId + 2];
                        foreach (ReplyInfo oldri in RendezVous)
                            Ids[oldri.reply_waitid] = true;
                        int newId = 0;
                        while (Ids[newId])
                            ++newId;
                        if (newId > maxRdvId)
                            maxRdvId = newId;
                        ri.reply_waitid = newId;
                        RendezVous.Add(ri);
                    }
                }
            }
            return (ri);
        }

        // destList is only non-null for subset Query operations
        internal static void awaitReplies(ReplyInfo ri, Group g, Timeout timeout, List<Address> destList)
        {
            ILock theILock = null;
            using (new LockAndElevate(Isis.RIPLock))
                if (destList != null)
                {
                    foreach (Address dest in destList)
                    {
                        int i = ri.rdvView.GetRawRankOf(dest);
                        if (i != -1 && Isis.RIPList.Contains(ri.rdvView.members[i]) && ri.gotReply[i] == false)
                            ri.UpdateRepliesWanted(i);
                    }
                }
                else for (int i = 0; i < ri.rdvView.members.Length; i++)
                        if (ri.gotReply[i] == false && Isis.RIPList.Contains(ri.rdvView.members[i]))
                            ri.UpdateRepliesWanted(i);
            using (new LockAndElevate(ri.Lock))
            {
                if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                    Isis.WriteLine("BEFORE rdvWait[" + ri.ricntr + "]: " + ri.rdvVid + ":" + ri.rdvMid + " in group " + ri.gaddr + ".... ri.wanted=" + ri.replies_wanted + ", ri.replies_received=" + ri.replies_received);
                if (ri.replies_wanted > 0)
                    ri.ilock = ILock.Barrier(ILock.LLREPLY, ri.reply_waitid);
                theILock = ri.ilock;
            }
            if (theILock != null)
            {
                int tid = Isis.OnTimerThread(Math.Max(100, timeout.when), (TimerCallback)delegate() { try { rdvTimeout(ri, timeout.action, timeout.origin); } catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); } });
                ri.tid = tid;
                if ((g.flags & Group.G_TERMINATING) == 0)
                    theILock.BarrierWait(g);
                Isis.TimerCancel(tid);
            }


            if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                Isis.WriteLine("AFTER rdvWait[" + ri.ricntr + "]: " + ri.rdvVid + ":" + ri.rdvMid + " in group " + ri.gaddr + ".... ri.wanted=" + ri.replies_wanted + ", ri.replies_received=" + ri.replies_received);

            using (new LockAndElevate(RendezVousLock))
            using (new LockAndElevate(ri.Lock))
                RendezVous.Remove(ri);

            using (new LockAndElevate(g.ViewHandlers.vhListLock))
                g.ViewHandlers.vhList.Remove(ri.vhcb);

            if (ri.replies_received == -1)
                throw new IsisAbortReplyException(ri.theReason);

            return;
        }

        internal static void gotReply(Msg rm)
        {
            if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                Isis.WriteLine("In gotReply " + rm);
            if ((rm.flags & Msg.FRAGGED) != 0)
            {
                rm.flags |= Msg.HASREPLY;
                Group.deFragRdv(Isis.ISISMEMBERS, rm);
                return;
            }
            using (new LockAndElevate(RendezVousLock))
            {
                object[] obs = Msg.BArrayToObjects(rm.payload);
                if (obs.Length < 5 || obs.Length > 6 ||
                    !obs[0].GetType().Equals(typeof(int)) || !obs[1].GetType().Equals(typeof(int)) || !obs[2].GetType().Equals(typeof(int)) ||
                    !obs[3].GetType().Equals(typeof(bool)) || !obs[4].GetType().Equals(typeof(bool)) ||
                    (obs.Length == 6 && !obs[5].GetType().Equals(typeof(byte[])) && !obs[5].GetType().Equals(typeof(string))))
                    throw new IsisException("gotReply: misformatted RT_REPLY object");
                int idx = 0;
                int rtype = (int)obs[idx++];
                int vid = (int)obs[idx++];
                int msgid = (int)obs[idx++];
                bool asOracle = (bool)obs[idx++];
                bool enciphered = (bool)obs[idx++];
                object theReply = null;
                if (obs.Length > 5)
                    theReply = (object)obs[idx];
                ReplyInfo theRi = null;
                foreach (ReplyInfo ri in RendezVous)
                {
                    using (new LockAndElevate(ri.Lock))
                    {
                        if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                            Isis.WriteLine("Scanning RI structure: " + ri.gaddr + "//" + ri.rdvVid + ":" + ri.rdvMid + ", I'm looking for " + rm.dest + "//" + vid + ":" + msgid);
                        if (ri.gaddr.Equals(rm.dest) && ri.rdvVid == vid && ri.rdvMid == msgid && ri.queryByOracle == asOracle && (theRi == null || theRi.uid < ri.uid))
                            theRi = ri;
                    }
                }
                if (theRi != null)
                {
                    if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                        Isis.WriteLine("... a match!");
                    using (new LockAndElevate(theRi.Lock))
                        doGotReply(theRi, rm.dest, rm.sender, rtype, vid, msgid, enciphered, theReply);
                    return;
                }
                // Falls through if the reply just isn't expected
                if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                    Isis.WriteLine("WARNING: gotReply not sure what to do with reply: " + rm);
                return;
            }
        }

        // Called with a lock on ri
        internal static void doGotReply(ReplyInfo ri, Address gaddr, Address sender, int rtype, int vid, int mid, bool enciphered, object theReply)
        {
            bool found = false;
            int i = ri.rdvView.GetRawRankOf(sender);
            if (i != -1)
            {
                if (ri.replies_wanted > 0 && ri.gotReply[i] == false)
                {
                    if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                        Isis.WriteLine("In doGotReply with rtype " + rtype + " from target " + i);
                    switch (rtype)
                    {
                        case Group.RT_REPLY:
                            if (enciphered != ri.enciphered)
                            {
                                Isis.WriteLine("WARNING: rejecting a reply: mismatched cipher status");
                                return;
                            }
                            ri.replies_received++;
                            ri.rdvReplies.Add((byte[])theReply);
                            if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                                Isis.WriteLine("ISAREPLY: currently need " + ri.replies_wanted + ", this was number " + ri.replies_received);
                            ri.UpdateRepliesWanted(i);
                            if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                                Isis.WriteLine("[1]: ri.wanted=" + ri.replies_wanted + ", data reply from " + sender);
                            found = true;
                            if (!ri.gaddr.Equals(gaddr))
                                throw new IsisException("doGotReply");
                            break;

                        case Group.RT_ABORT:
                            ri.theReason = (string)theReply;
                            ri.replies_wanted = 0;
                            ri.replies_received = -1;
                            if (ri.ilock != null)
                            {
                                ri.ilock.BarrierRelease(1);
                                ri.ilock = null;
                            }
                            if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                                Isis.WriteLine("[2]: ri.wanted=" + ri.replies_wanted + ", abortreply from " + sender);
                            found = true;
                            break;

                        case Group.RT_NULL:
                            ri.UpdateRepliesWanted(true, i);
                            if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                                Isis.WriteLine("[3]: ri.wanted=" + ri.replies_wanted + ", nullreply from " + sender);
                            found = true;
                            break;
                    }
                }
            }
            if (found == false && (IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
            {
                string rig = " ";
                for (i = 0; i < ri.rdvView.members.Length; i++)
                    rig += ri.gotReply[i] ? "+ " : "- ";
                Isis.WriteLine("**** WARNING IGNORING A REPLY, I was looking for " + sender + " in " + Address.VectorToString(ri.rdvView.members) + ", ri.gotReply [" + rig + "], ri.replies_wanted " + ri.replies_wanted);
            }
        }

        internal static void doNoteFailure(Address who)
        {
            using (new LockAndElevate(RendezVousLock))
            {
                foreach (ReplyInfo ri in RendezVous)
                {
                    using (new LockAndElevate(ri.Lock))
                    {
                        int i = ri.rdvView.GetRawRankOf(who);
                        if (i != -1 && ri.gotReply[i] == false)
                        {
                            ri.UpdateRepliesWanted(i);
                            if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                                Isis.WriteLine("[4]: ri.wanted=" + ri.replies_wanted + ", reply from " + who);
                            break;
                        }
                    }
                }
            }
        }

        internal static void InterruptReplyWaits(Group g)
        {
            using (new LockAndElevate(RendezVousLock))
            {
                foreach (ReplyInfo ri in RendezVous)
                {
                    if (!ri.gaddr.Equals(g.gaddr))
                        continue;
                    using (new LockAndElevate(ri.Lock))
                    {
                        if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                            Isis.WriteLine("[terminate] ri=" + ri);
                        ri.theReason = "g.Terminate";
                        ri.replies_wanted = 0;
                        ri.replies_received = 0;
                        if (ri.ilock != null)
                        {
                            ri.ilock.BarrierRelease(1);
                            ri.ilock = null;
                        }
                    }
                }
            }
        }

        internal static void rdvTimeout(ReplyInfo ri, int action, string origin)
        {
            List<Address> hasFailed = new List<Address>();
            using (new LockAndElevate(ri.Lock))
            {
                for (int i = 0; i < ri.rdvView.members.Length && ri.replies_wanted > 0; i++)
                    if (ri.gotReply[i] == false)
                    {
                        switch (action)
                        {
                            case Timeout.TO_NULLREPLY:
                                ri.UpdateRepliesWanted(i);
                                if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                                    Isis.WriteLine("[5]: ri.wanted=" + ri.replies_wanted + ", timeout(action=nullreply) " + ri.rdvView.members[i]);
                                break;

                            case Timeout.TO_ABORTREPLY:
                                ri.theReason = "timeout";
                                ri.replies_wanted = 0;
                                ri.replies_received = -1;
                                if (ri.ilock != null)
                                {
                                    ri.ilock.BarrierRelease(1);
                                    ri.ilock = null;
                                }
                                if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                                    Isis.WriteLine("[6]: ri.wanted=" + ri.replies_wanted + ", timeout(action=abortreply) " + ri.rdvView.members[i]);
                                break;

                            case Timeout.TO_FAILURE:
                                hasFailed.Add(ri.rdvView.members[i]);
                                break;
                        }
                    }
            }
            if (hasFailed.Count() == 0)
                return;
            string where = "(from timeout";
            if (origin != null)
                where += "/" + origin;
            foreach (Address who in hasFailed)
                Isis.NodeHasFailed(who, where + ": <" + ri.gname + ">:" + ri.rdvVid + ":" + ri.rdvMid + "; delay=" + ri.delay + ")", false);
            using (new LockAndElevate(ri.Lock))
            {
                for (int i = 0; i < ri.rdvView.members.Length; i++)
                    if (ri.gotReply[i] == false)
                        ri.UpdateRepliesWanted(i);
            }
        }
    }

    internal class FlowControl
    {
        internal static LockObject FCLock = new LockObject(false, "FClock");
        internal static long MsgCountNum;                       // Counts messages that the system is "holding"; numerator
        internal static long MsgCountDenom = 1;                 // Denominator
        internal volatile static int Waiting;

        internal static long GCD(long a, long b)
        {
            if (a < 0) a = -a;
            if (b < 0) b = -b;
            while (a != 0 && b != 0)
            {
                if (a > b)
                    a %= b;
                else
                    b %= a;
            }
            if (a == 0)
                return b;
            else
                return a;
        }

        private static void UpdateMsgCount(int num, int denom)
        {
            long tmpNum = (MsgCountNum * denom + num * MsgCountDenom);
            long tmpDenom = MsgCountDenom * denom;
            long gcd = GCD(tmpNum, tmpDenom);
            if (gcd != 1)
            {
                tmpNum /= gcd;
                tmpDenom /= gcd;
            }
            MsgCountNum = tmpNum;
            MsgCountDenom = tmpDenom;
            if (MsgCountNum < 0)
                throw new IsisException("UpdateMsgCount");
        }

        internal static void FCStartSend(int num, int denom)
        {
            using (new LockAndElevate(FCLock))
                UpdateMsgCount(num, denom);
            if ((IsisSystem.Debug & IsisSystem.FLOWCONTROL) != 0)
                Isis.WriteLine("After FCStartSend(" + num + "/" + denom + ")");
        }

        internal static void FCEndSend(int num, int denom)
        {
            bool firstLoop = true;
            int howMany;
            do
            {
                howMany = 0;
                using (new LockAndElevate(FCLock))
                {
                    if (firstLoop && num > 0)
                        UpdateMsgCount(-num, denom);
                    firstLoop = false;
                    if (MsgCountNum / MsgCountDenom < Isis.ISIS_ASYNCMTOTALLIMIT && Waiting > 0)
                    {
                        howMany = Math.Min(Waiting, (int)(Isis.ISIS_ASYNCMTOTALLIMIT - MsgCountNum / MsgCountDenom));
                        Waiting -= howMany;
                    }
                }
                if (howMany > 0)
                    ILock.Barrier(ILock.LLWAIT, ILock.LFLOWCNTRL).BarrierRelease(howMany);
            }
            while (howMany > 0);
            if ((IsisSystem.Debug & IsisSystem.FLOWCONTROL) != 0)
                Isis.WriteLine("After FCEndSend(" + num + "/" + denom + ")");
        }

        internal static void FCBarrierCheck()
        {
            FCBarrierCheck(null);
        }

        internal static void FCBarrierCheck(List<Address> dests)
        {
            ReliableSender.RemoteBacklogWait(dests);

            bool mustBlock = (Backlog(null) >= Isis.ISIS_ASYNCMTOTALLIMIT);
            mustBlock |= Group.Undelivered() > 64 || Group.nInUDPTunnel() > 32 || Group.nInIPMCTunnel() > 32 || ReliableSender.P2PSequencer.CountP2PSeqn() > 32;
            using (new LockAndElevate(FCLock))
            {
                if ((IsisSystem.Debug & IsisSystem.FLOWCONTROL) != 0)
                    Isis.WriteLine("Before FCBarrierCheck[" + Thread.CurrentThread.ManagedThreadId + "]: " + (mustBlock ? " WILL BLOCK " : " WON'T BLOCK ") + GetState());
                if (!mustBlock)
                    return;
                else
                    ++Waiting;
            }
            ILock.Barrier(ILock.LLWAIT, ILock.LFLOWCNTRL).BarrierWait();
            if ((IsisSystem.Debug & IsisSystem.FLOWCONTROL) != 0)
                Isis.WriteLine("After FCBarrierCheck(WAITED!)[" + Thread.CurrentThread.ManagedThreadId + "]: " + GetState());
        }

        internal static int Backlog(Group g)
        {
            if (IsisSystem.shuttingDown)
                return 0;
            int bl = CountBacklogs();
            if (g != null)
                g.MaxBacklogSent = Math.Max(g.MaxBacklogSent, bl);
            return bl + ReliableSender.P2PSequencer.CountP2PSeqn() + (int)(MsgCountNum / MsgCountDenom);
        }

        private static int CountBacklogs()
        {
            int gcnt = 0;
            List<Group> myCopy = Group.IsisGroupsClone();
            foreach (Group g in myCopy)
            {
                using (new LockAndElevate(g.UnstableLock))
                    if (g.Unstable != null)
                        gcnt += g.Unstable.Count();
                using (new LockAndElevate(g.ToDoLock))
                    if (g.ToDo != null)
                        gcnt += g.ToDo.Count();
                using (new LockAndElevate(g.PendingQueueLock))
                    if (g.PendingQueue != null)
                        for (int n = 0; n < g.PendingQueue.Length; n++)
                            if (g.PendingQueue[n] != null)
                                gcnt += g.PendingQueue[n].Count();
                using (new LockAndElevate(g.OutOfOrderQueueLock))
                    gcnt += g.OutOfOrderQueue.Count();
                using (new LockAndElevate(g.CausalOrderListLock))
                    gcnt += g.CausalOrderList.Count();
                gcnt += (BoundedBuffer.FullSlots(g.incomingSends) + BoundedBuffer.FullSlots(g.incomingP2P)) / 2;
            }
            gcnt += ReliableSender.LBBcnt() / 2;
            return gcnt;
        }

        internal static string GetState()
        {
            int fs = (ReliableSender.RecvBB == null) ? 0 : ReliableSender.RecvBB.fullSlots;
            return "Flow Control: " + MsgCountNum / MsgCountDenom + "+" + (MsgCountNum % MsgCountDenom) + "/" + MsgCountDenom + " + " + CountBacklogs() + " outgoing/unstable/on-todo-list, " +
                ReliableSender.P2PSequencer.CountP2PSeqn() + " on P2P or callback queues, " +
                fs + " full RecvBB slots, " + Group.nInUDPTunnel() + " in UDP tunnel, " + Group.nInIPMCTunnel() + " in IPMC tunnel, remote backlog=" + ReliableSender.RemoteBacklogCount() +
                "\r\n              " + Waiting + " threads waiting on local congestion, " + ReliableSender.rWaiting + " waiting on remote congestion\r\n";
        }
    }

    internal class ReliableSender
    {
        internal class msgWaiter
        {
            internal int threshold;
            internal Semaphore waitSema;

            internal msgWaiter(int t, Semaphore w) { threshold = t; waitSema = w; }
        }

        internal class activeMsg
        {
            internal Group g;
            internal int vid;
            internal int msgid;
            internal int StableCnt;          // Counts number of acks received
            internal int SentCnt;            // This is how many destinations it had in the first place
            internal List<msgWaiter> Waiters;         // When non-null, lists threads waiting on this message

            internal activeMsg(Group g, int vid, int msgid)
            {
                this.g = g; this.vid = vid; this.msgid = msgid;
            }
        }

        internal static List<activeMsg> activeMsgs = new List<activeMsg>();
        internal static LockObject activeMsgLock = new LockObject("activeMsgLock");

        internal class MsgDesc
        {
            internal byte type;
            internal Address dest;
            internal bool done;
            internal bool isp2p;
            internal bool isLarge;
            internal bool alreadyCleanedUp;
            internal Group group;
            internal int UID;
            internal int MsgVid;
            internal int MsgId;
            internal byte[] theBytes;
            internal int retryCnt;
            internal int retryLimit;
            internal int retryDelay;
            internal long remulticastTime;
            internal long resendTime;
            internal long firstSentAt;
            internal long lastSentAt;
            internal int num;
            internal int denom;
            internal bool localSender;
            internal Action p2pdcb;
            internal List<Semaphore> CPSList = new List<Semaphore>();
            internal activeMsg am;

            internal MsgDesc(byte type, int limit, Group group, Address dest, bool isp2p, bool isLarge, int uid, byte[] buffer, int vid, int MsgID, int rtd, int num, int denom, bool localSender, Action dcb, activeMsg am)
            {
                this.firstSentAt = this.lastSentAt = Isis.NOW();
                this.remulticastTime = Isis.NOW() + rtd;
                this.type = type;
                this.retryLimit = limit;
                this.dest = dest;
                this.done = false;
                this.isp2p = isp2p || (!dest.isNull() && !dest.isGroupAddress());
                this.isLarge = isLarge;
                this.group = group;
                this.UID = uid;
                this.MsgVid = vid;
                this.MsgId = MsgID;
                this.theBytes = buffer;
                this.retryDelay = Math.Max(50, rtd);
                this.num = num;
                this.denom = denom;
                this.localSender = localSender;
                this.p2pdcb = dcb;
                this.am = am;
                FlowControl.FCStartSend(num, denom);
                if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                    using (new LockAndElevate(ackInfoLock))
                        ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: Remember " + dest + "::" + MsgVid + ":" + MsgId + " -- UID " + uid + "\r\n");
            }

            public override string ToString()
            {
                return dest + "::" + MsgVid + ":" + MsgId;
            }
        }

        internal static LinkedList<MsgDesc> PendingSendBuffer = new LinkedList<MsgDesc>();
        internal static LinkedList<MsgDesc> LgPendingSendBuffer = new LinkedList<MsgDesc>();
        internal static LockObject PendingSendBufferLock = new LockObject("PendingSendBufferLock");


        internal static string GetState()
        {
            string s = "Pending Send Buffer: \r\n";
            List<MsgDesc> ps = new List<MsgDesc>(), pls = new List<MsgDesc>();
            using (new LockAndElevate(PendingSendBufferLock))
            {
                foreach (MsgDesc md in PendingSendBuffer)
                    ps.Add(md);
                foreach (MsgDesc md in LgPendingSendBuffer)
                    pls.Add(md);
            }
            foreach (MsgDesc md in ps)
                s += " Dest " + md.dest + (md.localSender ? "" : "(forwarded)") + "<" + (md.group == null ? "p2p" : md.group.gname) + ">, counts as " + md.num + "/" + md.denom + " for flow control, msgid " + md.MsgVid + ":" + md.MsgId +
                    ", uid " + md.UID + ", first sent at " + Isis.TimeToString(md.firstSentAt) + "... retry at " + Isis.TimeToString(md.remulticastTime) + ", retryCnt " + md.retryCnt + Msg.PPayload(md.theBytes) + "\r\n";
            s += "Large-Group Pending Send Buffer:\r\n";
            foreach (MsgDesc md in pls)
                s += " Dest " + md.dest + (md.localSender ? "" : "(forwarded)") + ", counts as " + md.num + "/" + md.denom + " for flow control, msgid " + md.MsgVid + ":" + md.MsgId +
                    ", uid " + md.UID + " first sent at " + Isis.TimeToString(md.firstSentAt) + ", last sent at " + Isis.TimeToString(md.lastSentAt) + ", retry at " + Isis.TimeToString(md.remulticastTime) + ", retryCnt " + md.retryCnt + Msg.PPayload(md.theBytes) + (md.p2pdcb == null ? " (no callback)" : " (callback on completion)") + "\r\n";
            s += P2PSequencer.GetState();
            return s;
        }

        internal static void CheckLen(byte[] buffer)
        {
            if (buffer.Length > Isis.ISIS_MAXMSGLEN)
                throw new IsisException("Trying to send an object of size " + buffer.Length + " yet Isis_MAXMSGLEN is " + Isis.ISIS_MAXMSGLEN);
        }

        internal static int getPendingP2PCount(List<Address> dests)
        {
            int ms = 0;
            List<P2PSequencer> currentP2PDests = new List<P2PSequencer>();
            using (new LockAndElevate(P2PSequencer.PSListLock))
                if (dests == null)
                    currentP2PDests = P2PSequencer.PSIndex.Values.ToList<P2PSequencer>();
                else foreach (var dest in dests)
                    {
                        P2PSequencer p2ps;
                        if (P2PSequencer.PSIndex.TryGetValue(dest, out p2ps))
                            using (new LockAndElevate(p2ps.Lock))
                                currentP2PDests.Add(p2ps);
                    }
            foreach (P2PSequencer p2ps in currentP2PDests)
                ms = Math.Max(ms, getRBacklog(p2ps));
            return ms;
        }

        internal static int getRBacklog(P2PSequencer p2ps)
        {
            int gap;
            using (new LockAndElevate(P2PSequencer.PSListLock))
                gap = p2ps.remoteBacklog;
            using (new LockAndElevate(PendingSendBufferLock))
            {
                foreach (MsgDesc md in PendingSendBuffer)
                    if (md.dest.Equals(p2ps.dest) && md.isp2p)
                        gap = Math.Max(gap, p2ps.outSeqn - md.MsgId);
            }
            return gap;
        }

        internal static void noteRemoteBacklog(Address who, int remoteBacklog)
        {
            if (who.isMyAddress())
                return;
            using (new LockAndElevate(P2PSequencer.PSListLock))
            {
                P2PSequencer p2ps;
                if (P2PSequencer.PSIndex.TryGetValue(who, out p2ps))
                    using (new LockAndElevate(p2ps.Lock))
                        p2ps.remoteBacklog = remoteBacklog;
            }
        }

        internal class AddrAndGroup
        {
            internal Address who;
            internal Address gaddr;

            internal AddrAndGroup(Address p, Address g)
            {
                who = p;
                gaddr = g;
            }

            public override bool Equals(object obj)
            {
                if (obj == null || !obj.GetType().Equals(typeof(AddrAndGroup)))
                    return false;
                return who.Equals(((AddrAndGroup)obj).who) && gaddr.Equals(((AddrAndGroup)obj).gaddr);
            }

            public override int GetHashCode()
            {
                return who.GetHashCode() * 37 + gaddr.GetHashCode();
            }
        }

        internal static int getPendingIPMCCount()
        {
            using (new LockAndElevate(PendingSendBufferLock))
                foreach (MsgDesc md in PendingSendBuffer)
                    if (md.localSender && !md.isp2p)
                        return ReliableSender.Counter - md.UID;
            return 0;
        }

        internal static int getRIPMCBacklog(AddrAndGroup ag)
        {
            int sum = 0;
            List<int> WorkingVidList = new List<int>();
            using (new LockAndElevate(PendingSendBufferLock))
            {
                foreach (MsgDesc md in PendingSendBuffer)
                    if (md.dest.Equals(ag.who) && !md.isp2p && md.group != null && md.group.gaddr.Equals(ag.gaddr) && (md.MsgVid != 0 || md.MsgId != -1))
                        if (!WorkingVidList.Contains(md.MsgVid))
                            WorkingVidList.Add(md.MsgVid);
                foreach (int workingVid in WorkingVidList)
                {
                    // For each active dest, for each group in which it sends, for each view with active messages, compute the number of active messages in the PSB
                    int maxMsgid = int.MinValue, minMsgid = int.MaxValue;
                    foreach (MsgDesc md in PendingSendBuffer)
                        if (md.MsgVid == workingVid && md.dest.Equals(ag.who) && !md.isp2p && md.group != null && md.group.gaddr.Equals(ag.gaddr))
                        {
                            minMsgid = Math.Min(md.MsgId, minMsgid);
                            maxMsgid = Math.Max(md.MsgId, maxMsgid);
                        }
                    if (maxMsgid != int.MinValue)
                        sum += maxMsgid - minMsgid + 1;
                }
            }
            return sum;
        }

        internal static void CleanLgCallbacks(Group g)
        {
            using (new LockAndElevate(PendingSendBufferLock))
            {
                foreach (MsgDesc lgmd in LgPendingSendBuffer)
                    if (lgmd.p2pdcb != null && lgmd.group == g)
                        lgmd.p2pdcb = null;
            }
        }

        internal static void Shutdown()
        {
            List<Group> IG = Group.IsisGroupsClone();
            foreach (Group g in IG)
                if (g.GroupOpen)
                    g.Flush();
            my_acksocket.Close();
            my_p2psocket.Close();
            TCPShutdown();
        }

        internal static LockObject rWaitLock = new LockObject("rWaitLock");
        internal static int rWaiting;

        internal static void RemoteBacklogWait(List<Address> dests)
        {
            while (!IsisSystem.IsisWasActive)
                Thread.Sleep(250);
            while (IsisSystem.IsisActive)
            {
                IsisSystem.RTS.ThreadCntrs[14]++;
                int backlog = RemoteBacklogCount(dests);
                if (backlog < Isis.ISIS_MAXRBACKLOG)
                    return;
                using (new LockAndElevate(rWaitLock))
                    ++rWaiting;
                ILock.Barrier(ILock.LLWAIT, ILock.LFLOWCNTRLB).BarrierWait();
                using (new LockAndElevate(rWaitLock))
                    --rWaiting;
            }
        }

        internal static int RemoteBacklogCount()
        {
            return RemoteBacklogCount(null);
        }

        internal static int RemoteBacklogCount(List<Address> dests)
        {
            int rbc = ReliableSender.getPendingP2PCount(dests) + ReliableSender.getPendingIPMCCount();
            return rbc;
        }

        internal static void CompletePendingSends(Group g, int cpscnt, Semaphore CPSSema)
        {
            g.CPSSema = CPSSema;
            while (cpscnt > 0)
            {
                CPSSema.WaitOne();
                --cpscnt;
            }
            g.CPSSema = null;
        }

        internal static void PendingSendCleanup(Address[] goodbye)
        {
            PendingSendCleanup(null, goodbye);
        }

        internal static void PendingSendCleanup(Group g, Address[] goodbye)
        {
            List<MsgDesc> cleanup = new List<MsgDesc>();
            using (new LockAndElevate(PendingSendBufferLock))
            {
                foreach (MsgDesc md in PendingSendBuffer)
                {
                    if (md.isp2p)
                        continue;
                    bool shouldRemove = false;
                    if (goodbye == null)
                        shouldRemove = true;
                    else foreach (Address a in goodbye)
                            if (md.dest.Equals(a) && (g == null || md.group == g))
                            {
                                shouldRemove = true;
                                break;
                            }
                    if (shouldRemove)
                        cleanup.Add(md);
                }
            }
            foreach (MsgDesc md in cleanup)
                GotAnAck(md.dest, md.UID);
        }

        internal static void lgPendingSendCleanup(Group g)
        {
            List<MsgDesc> cleanup = new List<MsgDesc>();
            using (new LockAndElevate(PendingSendBufferLock))
            {
                LinkedList<MsgDesc> newPSB = new LinkedList<MsgDesc>();
                foreach (MsgDesc lgmd in LgPendingSendBuffer)
                    if (lgmd.group != g && !lgmd.dest.Equals(g.gaddr))
                        newPSB.AddLast(lgmd);
                    else
                        cleanup.Add(lgmd);
            }
            int cnt = cleanup.Count();
            foreach (MsgDesc lgmd in cleanup)
            {
                lgmd.done = true;
                FlushingRemove(lgmd, "lgmd.lgPendingSendCleanup", --cnt == 0);
            }
        }

        internal static void LgAdd(Group g, Msg m, byte type)
        {
            using (new LockAndElevate(Group.GroupRIPLock))
            {
                if (Group.GroupRIPList.Contains(g.gaddr))
                    return;
            }
            using (new LockAndElevate(PendingSendBufferLock))
            {
                bool copyAll = false;
                LinkedList<MsgDesc> newLgPendingSendBuffer = new LinkedList<MsgDesc>();
                foreach (MsgDesc lgmd in LgPendingSendBuffer)
                    if (lgmd.group == g && lgmd.MsgVid == m.vid && lgmd.MsgId == m.msgid)
                        return;
                    else if (lgmd.group != g || copyAll || lgmd.MsgVid < m.vid || lgmd.MsgId < m.msgid)
                        newLgPendingSendBuffer.AddLast(lgmd);
                    else
                    {
                        newLgPendingSendBuffer.AddLast(new MsgDesc(type, 5, g, g.gaddr, false, true, m.msgid, m.offWire, m.vid, m.msgid, 0, 1, 5, false, null, null));
                        newLgPendingSendBuffer.AddLast(lgmd);
                        copyAll = true;
                    }
                if (!copyAll)
                    newLgPendingSendBuffer.AddLast(new MsgDesc(type, 5, g, g.gaddr, false, true, m.msgid, m.offWire, m.vid, m.msgid, 0, 1, 5, false, null, null));
                LgPendingSendBuffer = newLgPendingSendBuffer;
            }
        }

        internal static Thread AckThread;
        internal static Thread ReplyThread;
        internal static Socket my_acksocket;
        internal static Socket my_p2psocket;

        internal static Socket SetSocketUp(ref Socket s, ref int port, RcvrDel rd, ref Thread t, ThreadPriority pri, string name)
        {
            if (Isis.ISIS_TCP_ONLY == false)
            {
                s = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
                s.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, 1);
                // Tryto keep these fairly small....
                try
                {
                    s.ReceiveBufferSize = (int)Math.Min(64 * Isis.ISIS_MAXMSGLEN, int.MaxValue);
                    s.SendBufferSize = (int)Math.Min(32 * Isis.ISIS_MAXMSGLEN, int.MaxValue);
                }
                catch (Exception e)
                {
                    Isis.WriteLine("When trying to set Send/Receive buffer size<" + name + "> threw exception " + e);
                }
                if (Isis.ISIS_UDPCHKSUM)
                    s.SetSocketOption(SocketOptionLevel.Udp, SocketOptionName.ChecksumCoverage, 1);

                IPEndPoint localEndPoint = new IPEndPoint(Isis.my_IPaddress, 0);
                if (Isis.ISIS_UNICAST_ONLY)
                {
                    if (name.Equals("ack-socket"))
                        port = Isis.ISIS_DEFAULT_PORTNOa;
                    else if (name.Equals("p2p-socket"))
                        port = Isis.ISIS_DEFAULT_PORTNOp;
                    try
                    {
                        localEndPoint.Port = port;
                        s.Bind(localEndPoint);
                    }
                    catch
                    {
                        localEndPoint.Port = 0;
                        s.Bind(localEndPoint);
                    }
                }
                else
                    s.Bind(localEndPoint);
                port = ((IPEndPoint)s.LocalEndPoint).Port;

#if !__MonoCS__
                // Clumsy mechanism to prevent forcible socket reset events when an Isis application exits
                uint IOC_IN = 0x80000000;
                uint IOC_VENDOR = 0x18000000;
                uint SIO_UDP_CONNRESET = IOC_IN | IOC_VENDOR | 12;

                s.IOControl((int)SIO_UDP_CONNRESET, new byte[] { Convert.ToByte(false) }, null);
#endif // !__MonoCS__
            }
            else
            {
                s = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
                if (name.Equals("ack-socket"))
                    port = Isis.ISIS_DEFAULT_PORTNOa;
                else if (name.Equals("p2p-socket"))
                    port = Isis.ISIS_DEFAULT_PORTNOp;
                IPEndPoint ep = new IPEndPoint(Isis.my_IPaddress, port);
                for (int retry = 0; retry < 5; retry++)
                {
                    try
                    {
                        s.Bind(ep);
                        break;
                    }
                    catch
                    {
                        int oldport = port;
                        port += Isis.my_address.pid.GetHashCode() % 1000;
                        if (port == oldport)
                            port += 1234;
                        ep = new IPEndPoint(Isis.my_IPaddress, port);
                        try { s.Bind(ep); break; }
                        catch { if (retry == 5) throw new IsisException("Unable to bind to ISIS port (tried " + oldport + " and then " + port + ")"); else Thread.Sleep(250); }
                    }
                }
            }
            if (!Isis.ISIS_TCP_ONLY)
                s.ReceiveTimeout = 2500;
            t = StartSocketReader(s, rd, pri, name);
            return s;
        }

        internal delegate void RcvrDel(object s);
        internal static Thread myP2PThread, myAckThread;

        internal static Thread StartSocketReader(Socket s, RcvrDel rd, ThreadPriority p, string name)
        {
            Thread t;
            t = new Thread((ParameterizedThreadStart)delegate(object o)
            {
                try
                {
                    while (!IsisSystem.IsisActive)
                        Thread.Sleep(250);
                    while (IsisSystem.IsisActive)
                    {
                        IsisSystem.RTS.ThreadCntrs[15]++;
                        rd(s);
                    }
                }
                catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                catch (ThreadInterruptedException) { }
                catch (Exception e)
                {
                    if (IsisSystem.IsisActive)
                    {
                        IsisSystem.WriteLine("P2P or ACK  SocketReader threw an exception: " + e);
                        throw new IsisException("P2P or ACK thread fault");
                    }
                }
                IsisSystem.ThreadTerminationMagic();
            });
            //t.Priority = p;
            t.Name = "Isis " + name + " socket reader";
            t.Start();
            if (s == my_acksocket)
                myAckThread = t;
            else
                myP2PThread = t;
            return t;
        }

        internal static void Init()
        {
            if (Isis.ISIS_TCP_ONLY)
                TCPTunnelSetup();
            if (my_p2psocket == null)
                my_p2psocket = SetSocketUp(ref my_p2psocket, ref Isis.my_address.p2pPort, ReliableSender.SReceive, ref ReplyThread, ThreadPriority.AboveNormal, "p2p-socket");
            if (my_acksocket == null)
                my_acksocket = SetSocketUp(ref my_acksocket, ref Isis.my_address.ackPort, AckSocketReader, ref AckThread, ThreadPriority.Highest, "ack-socket");
            if (Isis.ISIS_TCP_ONLY)
                ReliableSender.TCPListen();
        }

        internal static void StartGroupReader(Group g)
        {
            try
            {
                if (g.my_socket != null || (!g.isTrackingProxy && !g.GroupOpen && g.WasOpen))
                    return;
                int hash = Address.GroupNameHash(g.gname);
                g.myVirtIPAddr = Isis.CLASSD + Isis.ISIS_MCRANGE_LOW + hash;
                IPAddress theIPAddr = Isis.LastIPv4(MCMDSocket.PMCAddr(g.myVirtIPAddr));
                if (g == Isis.ORACLE)
                {
                    if (Isis.ISIS_UNICAST_ONLY == false)
                        g.myPhysIPAddr = g.myVirtIPAddr;
                    else
                        g.myPhysIPAddr = MCMDSocket.USEUNICAST;
                    g.my_socket = new MCMDSocket(g.gname, g.isTrackingProxy, g.myVirtIPAddr, g.myPhysIPAddr);
                }
                else
                    g.my_socket = new MCMDSocket(g.gname, g.isTrackingProxy, g.myVirtIPAddr);
                g.gaddr = new Address(theIPAddr, 0);
                if (g.isTrackingProxy)
                    return;
                g.groupReaderThread = new Thread((ParameterizedThreadStart)delegate(object o)
                {
                    try
                    {
                        while (IsisSystem.IsisActive && (g.GroupOpen || !g.WasOpen))
                        {
                            IsisSystem.RTS.ThreadCntrs[16]++;
                            ReliableSender.Receive(g);
                        }
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    IsisSystem.ThreadTerminationMagic();
                });
                g.groupReaderThread.Name = "Isis <" + g.gname + "> incoming thread";
                g.groupReaderThread.Start();
            }
            catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
        }

        internal static int[] getNetworkInterfaces(string[] which)
        {
            int minMTU = int.MaxValue;
            int nInterfaces = 0;
            int[] InterfaceList = new int[50];
            NetworkInterface[] nics = NetworkInterface.GetAllNetworkInterfaces();
            if (nics != null)
                foreach (NetworkInterface adapter in nics)
                {
                    if (which != null)
                    {
                        int idx;
                        for (idx = 0; idx < which.Length; idx++)
                            if (adapter.Name.Equals(which[idx]))
                                break;
                        if (idx == which.Length)
                            continue;
                    }
                    PhysicalAddress address = adapter.GetPhysicalAddress();
                    byte[] bytes = address.GetAddressBytes();
                    if ((IsisSystem.Debug & IsisSystem.INTERFACES) != 0)
                    {
                        string pas = "";
                        for (int i = 0; i < bytes.Length; i++)
                        {
                            pas += bytes[i].ToString("X2");
                            if (i != bytes.Length - 1)
                                pas += "-";
                        }
                        Isis.WriteLine("Success!  [" + adapter.Name + "]: Checking status=" + adapter.OperationalStatus +
                            ", supports multicast=" + adapter.SupportsMulticast +
                            ",  adapter.Supports(NetworkInterfaceComponent.IPv4)=" + adapter.Supports(NetworkInterfaceComponent.IPv4) +
                            ", physaddr=" + pas +
                            ", description=" + adapter.Description);
                    }
                    if (adapter.OperationalStatus != OperationalStatus.Up || adapter.Supports(NetworkInterfaceComponent.IPv4) == false || adapter.NetworkInterfaceType.ToString().Contains("Loopback"))
                        continue;
                    IPv4InterfaceProperties p = adapter.GetIPProperties().GetIPv4Properties();
                    if (p == null)
                        continue;
                    if ((IsisSystem.Debug & IsisSystem.INTERFACES) != 0)
                        Isis.WriteLine("  [" + adapter.Name + "]: got IPv4Properties!  index=" + p.Index + ", MTU=" + p.Mtu + ", forwards=" + p.IsForwardingEnabled);
                    minMTU = Math.Max(1400, Math.Min(p.Mtu, minMTU));
                    if (adapter.SupportsMulticast)
                        InterfaceList[nInterfaces++] = p.Index;
                    foreach (var addr in adapter.GetIPProperties().UnicastAddresses)
                        if (addr.Address.AddressFamily.Equals(AddressFamily.InterNetwork))
                            Isis.my_IPaddress = addr.Address;
                }
            Isis.ArrayResize<int>(ref InterfaceList, nInterfaces);
            if (minMTU != int.MaxValue)
                Isis.ISIS_MAXMSGLEN = minMTU << 4;
            return InterfaceList;
        }

        internal static Thread ReplayerThread;
        internal static Thread ResenderThread;
        internal static Semaphore ReplayerSleepLock = new Semaphore(0, int.MaxValue);
        internal static Semaphore ResenderSleepLock = new Semaphore(0, int.MaxValue);

        // Launches replayer and resender threads
        internal static void ResenderThreadLaunch()
        {
            ReplayerThread = new Thread(Replayer);
            ReplayerThread.Name = "Isis Replayer thread";
            ReplayerThread.Start();
            ResenderThread = new Thread(Resender);
            ResenderThread.Name = "Isis Resender thread";
            ResenderThread.Priority = ThreadPriority.AboveNormal;
            ResenderThread.Start();
        }

        internal static void TickleResenderThread()
        {
            ReplayerSleepLock.Release(1);
            ResenderSleepLock.Release(1);
        }

        private static Random rand = new Random();

        private static List<MsgDesc> Scramble(List<MsgDesc> list)
        {
            List<MsgDesc> newList = new List<MsgDesc>();
            MsgDesc[] mda = list.ToArray();
            for (int n = mda.Length - 1; n >= 0; n--)
            {
                int off = (n < 2) ? n : rand.Next(n + 1);
                newList.Add(mda[off]);
                mda[off] = mda[n];
            }
            return newList;
        }

        // Replays Stashed and ToDo lists
        internal static void Replayer()
        {
            while (!IsisSystem.IsisActive)
                Thread.Sleep(250);
            try
            {
                while (IsisSystem.IsisActive)
                {
                    IsisSystem.RTS.ThreadCntrs[17]++;
                    ReliableSender.P2PSequencer.CheckP2PSeqn();
                    IsisSystem.RTS.check();
                    if (Group.stashNonEmpty)
                        Group.stashCheck();
                    List<Group> theGroups = Group.IsisGroupsClone();
                    foreach (Group g in theGroups)
                    {
                        g.ReplayToDo();
                        g.CheckRetainedOpqINodes();
                        g.CheckCausalWaitQueue();
                    }
                    ReplayerSleepLock.WaitOne(1000);
                    string allCntrs = " ";
                    for (int ctr = 0; ctr < IsisSystem.RTS.ThreadCntrs.Length; ctr++)
                    {
                        if (IsisSystem.RTS.ThreadCntrs[ctr] > 10000)
                            allCntrs += ctr + ":" + IsisSystem.RTS.ThreadCntrs[ctr].ToString() + " ";
                        IsisSystem.RTS.ThreadCntrs[ctr] = 0;
                    }
                    if (allCntrs.Length > 1)
                        Isis.WriteLine("WARNING: Thread loop counter(s) advanced rapidly: " + allCntrs);
                }
            }
            catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
            IsisSystem.ThreadTerminationMagic();
        }

        // Be very careful with locking in this code; it needs to be blazingly fast and to never pause
        internal static void Resender()
        {
            try
            {
                while (!IsisSystem.IsisActive)
                    ResenderSleepLock.WaitOne(250);
                int rsCount = 0, szeroCount = 0;
                while (IsisSystem.IsisActive)
                {
                    IsisSystem.RTS.ThreadCntrs[18]++;
                    CleanSentNack();
                    Group.SendStability();
                    if (FlowControl.Waiting > 0)
                        FlowControl.FCEndSend(0, 1);
                    List<MsgDesc> psb, lgpsb, toRemove;
                    LinkedList<MsgDesc> newPSB, newLgPSB;
                    int sleep_until = 1000;
                    List<MsgDesc> destDied = new List<MsgDesc>();
                    using (new LockAndElevate(PendingSendBufferLock))
                    {
                        newPSB = new LinkedList<MsgDesc>();
                        psb = new List<MsgDesc>();
                        long now = Isis.NOW();
                        List<MsgDesc> toAdd = new List<MsgDesc>();
                        foreach (MsgDesc md in PendingSendBuffer)
                        {
                            bool send = true;
                            using (new LockAndElevate(Isis.RIPLock))
                                if (Isis.RIPList.Contains(md.dest))
                                {
                                    destDied.Add(md);
                                    continue;
                                }
                            if (md.group != null && !md.isp2p)
                                using (new LockAndElevate(Group.GroupRIPLock))
                                    if (Group.GroupRIPList.Contains(md.group.gaddr))
                                    {
                                        destDied.Add(md);
                                        continue;
                                    }
                            newPSB.AddLast(md);
                            foreach (MsgDesc mmd in psb)
                                if (!mmd.dest.Equals(md.dest))
                                    toAdd.Add(mmd);
                                else if ((mmd.UID < md.UID && mmd.retryCnt < md.retryCnt) || ((now - mmd.lastSentAt) > 5000 && now > md.resendTime))
                                {
                                    toAdd.Add(mmd);
                                    send = false;
                                    break;
                                }
                            if (send)
                                toAdd.Add(md);
                        }
                        foreach (MsgDesc nmd in toAdd)
                            psb.Add(nmd);
                        PendingSendBuffer = newPSB;
                        int npsb = psb.Count();
                        if (npsb < 10 && PendingSendBuffer.Count() > 5)
                            foreach (MsgDesc md in PendingSendBuffer)
                                if (now > md.remulticastTime && !psb.Contains(md))
                                {
                                    psb.Add(md);
                                    if (++npsb == 10)
                                        break;
                                }
                        lgpsb = new List<MsgDesc>();
                        newLgPSB = new LinkedList<MsgDesc>();
                        toRemove = new List<MsgDesc>();
                        using (new LockAndElevate(Group.GroupRIPLock))
                            foreach (MsgDesc md in LgPendingSendBuffer)
                            {
                                if (md.retryCnt < Isis.ISIS_MAXLGRETRIES)
                                    lgpsb.Add(md);
                                if (!md.isp2p && Group.GroupRIPList.Contains(md.dest))
                                    toRemove.Add(md);
                                else
                                    newLgPSB.AddLast(md);
                            }
                        LgPendingSendBuffer = newLgPSB;

                    }
                    psb = Scramble(psb);
                    lgpsb = Scramble(lgpsb);
                    int cnt = toRemove.Count() + destDied.Count();
                    foreach (MsgDesc md in destDied)
                    {
                        md.done = true;
                        FlushingRemove(md, "resender: found dest on RIPList", --cnt == 0);
                    }
                    foreach (MsgDesc md in toRemove)
                    {
                        md.done = true;
                        FlushingRemove(md, "resender: found dest on GroupRIPList", --cnt == 0);
                    }
                    foreach (MsgDesc md in psb)
                    {
                        long tryingFor = Isis.NOW() - md.firstSentAt;
                        if (Isis.NOW() < md.resendTime && tryingFor < Isis.ISIS_DEFAULTTIMEOUT)
                            continue;
                        if (!md.localSender)
                        {
                            md.done = true;
                            FlushingRemove(md, "Forwarded message, tried once", true);
                            sleep_until = 0;
                        }
                        else if (!HeardFromRecently(md.dest) && ((md.retryCnt >= md.retryLimit && tryingFor > Isis.ISIS_DEFAULTTIMEOUT * 2 / 3)) || (md.retryCnt == Isis.ISIS_MAXRETRIES && tryingFor > Isis.ISIS_DEFAULTTIMEOUT))
                        {
                            md.done = true;
                            FlushingRemove(md, "from resender.sendto because dest has failed", true);
                            Isis.NodeHasFailed(md.dest, "from resender.sendto for dest " + md.dest + ", ID " + md.UID + "/" + md.MsgVid + ":" + md.MsgId + " (retried for " + Isis.TimeToString(tryingFor) + " secs)", false);
                            sleep_until = 50;
                            break;
                        }
                        else
                        {
                            if (++rsCount > Isis.ISIS_ASYNCMTOTALLIMIT)
                            {
                                rsCount = 0;
                                if (!ResenderSleepLock.WaitOne(10))
                                    sleep_until = 0;
                            }
                            if ((IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0)
                                Isis.WriteLine("Resender.sendto[Now:" + Isis.NOW() + ", md.remulticastTime:" + md.remulticastTime + "]: dest " + md.dest + ", ID " + md.UID + "/" + md.MsgVid + ":" + md.MsgId + ", len " + md.theBytes.Length + " bytes");
                            if (md.retryCnt < Isis.ISIS_MAXRETRIES && Resend(md))
                                ++md.retryCnt;
                            md.retryDelay = Math.Min(Isis.ISIS_MIN2NDRTSEND, Math.Max(250, md.retryDelay << 1));
                            md.resendTime = Isis.NOW() + md.retryDelay;
                            if (md.retryDelay < sleep_until)
                                sleep_until = md.retryDelay;
                        }
                    }

                    foreach (MsgDesc lgmd in lgpsb)
                    {
                        if (Isis.NOW() < lgmd.remulticastTime || !lgmd.group.GroupOpen)
                            continue;
                        Group.tokenInfo theToken = lgmd.group.theToken;
                        if (theToken == null)
                            continue;
                        if (++rsCount > Isis.ISIS_ASYNCMTOTALLIMIT)
                        {
                            rsCount = 0;
                            if (!ResenderSleepLock.WaitOne(10))
                                sleep_until = 0;
                        }
                        if (((lgmd.retryCnt < 5 && (Isis.NOW() - lgmd.lastSentAt) > 1000) || (Isis.NOW() - lgmd.lastSentAt) > 5000) && ReMulticast(lgmd))
                            ++lgmd.retryCnt;
                        if (lgmd.retryDelay < sleep_until)
                            sleep_until = lgmd.retryDelay * lgmd.retryCnt;
                        lgmd.remulticastTime = Isis.NOW() + Math.Max(1000, Math.Max(500 + Math.Min(4, lgmd.retryCnt) * 250, lgmd.retryDelay * lgmd.retryCnt));
                    }
                    TCPListCheck();
                    if (sleep_until != 0)
                    {
                        szeroCount = 0;
                        ResenderSleepLock.WaitOne(Math.Min(1000, sleep_until));
                    }
                    else if (++szeroCount == 5)
                    {
                        // After sending a burst of 25 retransmissions, with 10ms pauses every 5 messages send, pause for 100ms
                        szeroCount = 0;
                        ResenderSleepLock.WaitOne(100);
                    }
                }
                if ((IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0)
                    Isis.WriteLine("IsisActive false: Resender exiting.");
            }
            catch (IsisShutdown) { }
            catch (SocketException) { if (IsisSystem.IsisActive) throw new IsisException("Isis terminating due to socket exception in resender thread"); }
            if (IsisSystem.IsisActive)
                throw new IsisException("Resender thread has crashed");
            IsisSystem.ThreadTerminationMagic();
        }

        internal static void P2PSend(Address dest, int port, byte[] data, int which)
        {
            IPEndPoint remoteEP = new IPEndPoint(dest.home, port);
            if (Isis.ISIS_TCP_ONLY)
                ReliableSender.TCPSendTo(dest, data, remoteEP, which);
            else
            {
                try
                {
                    if (which == RECVBB)
                    {
                        ReliableSender.CheckLen(data);
                        int bs = ReliableSender.my_p2psocket.SendTo(data, remoteEP);
                        using (new LockAndElevate(IsisSystem.RTS.Lock))
                        {
                            IsisSystem.RTS.UDPBsent += bs;
                            IsisSystem.RTS.UDPsent++;
                        }
                    }
                    else
                    {
                        ReliableSender.CheckLen(data);
                        ReliableSender.my_acksocket.SendTo(data, remoteEP);
                        using (new LockAndElevate(IsisSystem.RTS.Lock))
                            IsisSystem.RTS.ACKsent++;
                    }
                }
                catch { }
            }
        }


        internal static bool Resend(ReliableSender.MsgDesc md)
        {
            if (Isis.NOW() - md.lastSentAt < 150)
                return false;
            md.lastSentAt = Isis.NOW();
            if (Isis.ISIS_TCP_ONLY)
            {
                if (Isis.ISISMEMBERS != null && Isis.ISISMEMBERS.HasFirstView)
                    Isis.ISISMEMBERS.UDPTunnel(md.dest, md.retryCnt, md.theBytes, 0);
            }
            else
            {
                try
                {
                    IPEndPoint target = new IPEndPoint(md.dest.home, md.dest.p2pPort);
                    ReliableSender.CheckLen(md.theBytes);
                    int bs = ReliableSender.my_p2psocket.SendTo(md.theBytes, target);
                    if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                        using (new LockAndElevate(ackInfoLock))
                            ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: Resend to " + target + " for " + md.dest + " UID " + md.UID + "\r\n");
                    using (new LockAndElevate(IsisSystem.RTS.Lock))
                    {
                        IsisSystem.RTS.UDPBsent += bs;
                        IsisSystem.RTS.UDPsent++;
                    }
                }
                catch (IOException) { }
                catch (SocketException) { }
                catch (ObjectDisposedException) { throw new IsisShutdown("socket disposed"); }
            }
            return true;
        }

        internal static void ResendP2P(Address dest, ReliableSender.MsgDesc md)
        {
            if (Isis.ISIS_TCP_ONLY)
            {
                TunnelMsg(true, dest, md.theBytes, (md.group == null) ? Isis.ISISMEMBERS.gaddr : md.group.gaddr);
                return;
            }
            try
            {
                IPEndPoint remoteEP = new IPEndPoint(dest.home, dest.p2pPort);
                ReliableSender.CheckLen(md.theBytes);
                int bs = ReliableSender.my_p2psocket.SendTo(md.theBytes, remoteEP);
                using (new LockAndElevate(IsisSystem.RTS.Lock))
                {
                    IsisSystem.RTS.UDPBsent += bs;
                    IsisSystem.RTS.UDPsent++;
                }
            }
            catch { }
        }

        internal static bool ReMulticast(MsgDesc lgmd)
        {
            if ((Isis.NOW() - lgmd.lastSentAt < 500) || lgmd.group == null || lgmd.group.gaddr == null || lgmd.group.my_socket == null || lgmd.theBytes == null)
                return false;
            if (!lgMulticast(lgmd.group, lgmd.theBytes))
                return false;
            lgmd.lastSentAt = Isis.NOW();
            lgmd.retryDelay = Math.Min(5000, Math.Max(50, lgmd.retryDelay));
            lgmd.remulticastTime = Isis.NOW() + lgmd.retryDelay;
            return true;
        }

        internal static bool lgMulticast(Group g, byte[] ba)
        {
            if (g.myPhysIPAddr != MCMDSocket.USEUNICAST && !Isis.ISIS_TCP_ONLY)
                return g.my_socket.NBSendTo(ba);
            if (Isis.ISIS_TCP_ONLY || g.theView.members.Length > Isis.ISIS_MAXDIRECTSENDS)
            {
                Isis.ISISMEMBERS.IPMCTunnel(g.gaddr, ba);
                return true;
            }
            foreach (Address dest in g.theView.members)
            {
                if (dest.isMyAddress())
                    continue;
                IPEndPoint remoteEP = new IPEndPoint(dest.home, dest.p2pPort);
                ReliableSender.CheckLen(ba);
                int bs = my_p2psocket.SendTo(ba, remoteEP);
                using (new LockAndElevate(IsisSystem.RTS.Lock))
                {
                    IsisSystem.RTS.UDPBsent += bs;
                    IsisSystem.RTS.UDPsent++;
                }
            }
            return true;
        }

        internal class TCPSockInfo : IDisposable
        {
            internal IPEndPoint remoteEndPoint;
            internal int which;
            internal Socket theSocket;
            internal Address dest;
            internal LockObject rcvLock = new LockObject("rcvLock", ThreadPriority.Highest);
            internal Semaphore sndLock = new Semaphore(1, 1);
            internal LockObject siLock = new LockObject("siLock");
            internal BoundedBuffer TCPBB;
            internal Thread TCPSender;
            internal Thread TCPReceiver;
            internal long openedAt;
            internal long lastUsed;
            volatile internal bool closed = false;
            internal bool initiatedByMe = false;
            volatile internal int nwrites;
            volatile internal int nreads;
            internal int index;
            internal static int cntr;
            internal int lastISISMEMBERSViewDelta;

            internal TCPSockInfo(Address d, IPEndPoint ep, Socket s, int w, bool ibm)
            {
                remoteEndPoint = ((dest = d) == null) ? ep : AddressToIPEndpoint(d, w);
                lastUsed = openedAt = Isis.NOW();
                initiatedByMe = ibm;
                theSocket = s;
                index = ++cntr;
                TCPOverlaySetupSocket(s);
                which = w;
                string name = "TCPSender[index=" + index + "]" + d;
                TCPBB = new BoundedBuffer(name, 128, ILock.LLTCP, -1, -1, ThreadPriority.Highest);
                TCPSender = new Thread(delegate()
                {
                    try
                    {
                        TCPSender.Priority = ThreadPriority.Highest;
                        try
                        {
                            object toSend;
                            while ((toSend = TCPBB.get()) != null)
                                doSendAllBytes(this, (byte[])toSend);
                        }
                        catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); };
                        using (new LockAndElevate(TCPLock))
                        {
                            closed = true;
                            if (TCPList.Contains(this))
                                TCPList.Remove(this);
                            TCPBB.put(null);
                        }
                        using (new LockAndElevate(siLock))
                        {
                            try
                            {
                                if (theSocket != null)
                                {
                                    if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                                        Isis.WriteLine("Close dest=" + d + " " + TCPCnames[w] + ", byme=" + ibm + ", si.theSocket=" + theSocket.Handle);
                                    theSocket.Close();
                                }
                            }
                            catch { }
                        }
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    IsisSystem.ThreadTerminationMagic();
                });
                TCPSender.Name = name;
                TCPSender.Start();
            }

            public void Dispose()
            {
                Dispose(true);
            }

            private void Dispose(bool disposing)
            {
                if (disposing)
                    BoundedBuffer.unregister(TCPBB);
            }

            ~TCPSockInfo()
            {
                Dispose(true);
            }
        };

        // Puts TCP overloay into send immediate (non-Nagle) mode, disables non-blocking reads, and
        // sets them up with 2 seconds to "drain" any outgoing data after a close is sent.  Since we
        // close only idle sockets and only after many seconds, this is a long enough delay: all
        // the socket needs to send is the close itself
        internal static void TCPOverlaySetupSocket(Socket s)
        {
            try
            {
                s.NoDelay = true;
                s.Blocking = true;
                s.LingerState = new LingerOption(true, 2);
            }
            catch { }
        }

        /// <exclude>
        /// <summary>
        /// TCPhdr preceeds any incoming TCP message
        /// </summary>
        /// </exclude>
        [AutoMarshalled]
        public class TCPhdr
        {
            public byte TCPCommand;
            public int len;

            public TCPhdr()
            {
            }

            internal TCPhdr(byte c, int l)
            {
                TCPCommand = c;
                len = l;
            }
        }

        internal const int SEEKU = 0;
        internal const int ACKBB = 1;
        internal const int RECVBB = 2;

        internal static string[] TCPCnames = new string[] { "SEEKU", "Ack", "P2P" };

        internal const byte TCPC_SEEKU = 0;
        internal const byte TCPC_ACKCH = 1;
        internal const byte TCPC_P2PCH = 2;
        internal const byte TCPC_MSG = 3;
        internal const byte TCPC_CLOSE = 4;
        internal const byte TCPC_VDS = 5;

        internal static string[] TCPCommands = new string[] { "seek you", "ack-channel", "p2p channel", "msg", "close", "vds" };

        internal static int TCPHdrLen;

        internal static List<TCPSockInfo> TCPList = new List<TCPSockInfo>();
        internal static LockObject TCPLock = new LockObject("TCPLock", ThreadPriority.Highest);

        internal class AWPair
        {
            internal Address dest;
            internal int which;
            internal List<byte[]> toSend = new List<byte[]>();

            internal AWPair(Address a, int w, byte[] buf)
            {
                dest = a;
                which = w;
                if (buf != null)
                    toSend.Add(buf);
            }
        }

        internal static LockObject ConnectingLock = new LockObject("ConnectingLock");
        internal static List<AWPair> ConnectingTo = new List<AWPair>();

        internal static bool ConnectingToCheck(Address dest, byte[] buf, int which)
        {
            return ConnectingToCheck(dest, buf, which, false);
        }

        internal static bool ConnectingToCheck(Address dest, byte[] buf, int which, bool Add)
        {
            using (new LockAndElevate(ConnectingLock))
            {
                foreach (AWPair awp in ConnectingTo)
                    if (awp.which == which && ((awp.dest == null && dest == null) || (awp.dest != null && dest != null && awp.dest.Equals(dest))))
                    {
                        if (buf != null)
                            awp.toSend.Add(buf);
                        return true;
                    }
                if (Add && buf != null && dest != null && !dest.isNull())
                    ConnectingTo.Add(new AWPair(dest, which, buf));
            }
            return false;
        }

        internal static void ConnectingToDone(TCPSockInfo si)
        {
            List<byte[]> toSend = null;
            using (new LockAndElevate(ConnectingLock))
                foreach (AWPair awp in ConnectingTo)
                    if (awp.which == si.which && awp.dest.Equals(si.dest))
                    {
                        toSend = awp.toSend;
                        ConnectingTo.Remove(awp);
                        break;
                    }
            if (toSend == null || toSend.Count() == 0)
                return;
            new Thread(delegate()
            {
                try
                {
                    Thread.CurrentThread.Name = "Draining toSend for " + si.dest + ", mode " + TCPCnames[si.which];
                    foreach (byte[] buf in toSend)
                        TCPSendTo(si.dest, buf, si.remoteEndPoint, si.which);
                }
                catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                IsisSystem.ThreadTerminationMagic();
            }).Start();
        }

        // Normally, called with a fully known destination.  But if dest.isNull(), which occurs only during startup, retains no memory of connection
        internal static void TCPSendTo(Address dest, byte[] buf, IPEndPoint ipept, int which)
        {
            if (!Isis.ISIS_TCP_ONLY)
                throw new IsisException("TCPSendTo but not in ISIS_TCP_ONLY mode!");
            if (dest.isMyAddress() || ConnectingToCheck(dest, buf, which))
                return;
            TCPSockInfo tcpSI = FindTPCsi(dest, ipept, which);
            if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                Isis.WriteLine("TCPSendTo: dest=" + dest + ", command = " + TCPCnames[which] + ", endpoint=" + ipept + ", data.Length=" + buf.Length);
            try
            {
                if (tcpSI == null)
                {
                    TCPConnectTo(dest, ipept, which, false, buf);
                    return;
                }
                TCPhdr tcphdr = new TCPhdr(TCPC_MSG, buf.Length);
                using (new LockAndElevate(tcpSI.siLock))
                {
                    tcpSI.lastUsed = Isis.NOW();
                    tcpSI.nwrites++;
                }
                byte[] ha = Msg.toBArray(tcphdr);
                SendAllBytes(tcpSI, ha, buf);
                if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                    Isis.WriteLine("After TCPSnd<" + TCPCnames[which] + "> " + buf.Length);
            }
            catch (Exception e)
            {
                // During TCP_ONLY mode startup if a node tries to connect to itself we end up here
                if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                    Isis.WriteLine("TCP SendTo<" + TCPCnames[which] + "> " + buf.Length + " threw exception " + e);
                if (tcpSI != null)
                    using (new LockAndElevate(TCPLock))
                    {
                        tcpSI.TCPBB.put(null);
                        if (TCPList.Contains(tcpSI))
                            TCPList.Remove(tcpSI);
                    }
            }

            if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                Isis.WriteLine("tCPSendTo: After sending to " + ipept + ": " + TCPListState());
        }

        private static void SendAllBytes(TCPSockInfo tcpSI, byte[] hdr)
        {
            SendAllBytes(tcpSI, hdr, null);
        }

        private static void SendAllBytes(TCPSockInfo tcpSI, byte[] hdr, byte[] body)
        {
            if (tcpSI.sndLock.WaitOne(Isis.ISIS_DEFAULTTIMEOUT))
            {
                if (tcpSI.TCPBB.putWillBlock(2))
                    return;
                tcpSI.TCPBB.put(hdr);
                if (body != null)
                    tcpSI.TCPBB.put(body);
                tcpSI.sndLock.Release(1);
            }
            else
            {
                throw new SocketException(10035 /* WSAEWOULDBLOCK */);
            }
        }

        private static void doSendAllBytes(TCPSockInfo tcpSI, byte[] ha)
        {
            try
            {
                int nsent = 0;
                while (nsent < ha.Length)
                    nsent += tcpSI.theSocket.Send(ha, nsent, ha.Length - nsent, SocketFlags.None);
                using (new LockAndElevate(IsisSystem.RTS.Lock))
                {
                    IsisSystem.RTS.TCPsent++;
                    IsisSystem.RTS.TCPBsent += nsent;
                }
            }
            catch (ObjectDisposedException)
            {
                // Occurs if the local endpoint is closed just as a send is occuring
                // A consequence of the free-for-all during intialization, when we can end up with
                // replicated connections (A -> B and B -> A), then close one of them.  We can safely
                // ignore the error; any packets in transit get resent if they don't get through
                return;
            }
            catch (SocketException)
            {
                // Occurs if the remote endpoint crashes
                TCPClose(tcpSI, true);
                return;
            }
        }

        internal static IPEndPoint lastFound = null;
        internal static TCPSockInfo lasttcpSI = null;

        // Look up the destination.  If there are duplicate entries, favor the "smaller" one; if two are identical, favor the more recently used
        // The idea is that if a duplicate does arise, it will soon "age out" from lack of use
        private static TCPSockInfo FindTPCsi(Address dest, IPEndPoint ipept, int which)
        {
            IPEndPoint target;
            if (dest != null && !dest.isNull())
                target = AddressToIPEndpoint(dest, which);
            else
                return null;
            TCPSockInfo tcpSI = null;
            using (new LockAndElevate(TCPLock))
            {

                if (target == lastFound && lasttcpSI != null)
                    tcpSI = lasttcpSI;
                else
                {
                    foreach (TCPSockInfo tsi in TCPList)
                        if (tsi.remoteEndPoint.Equals(target) && (tcpSI == null || IPEndpointCompare(tcpSI.remoteEndPoint, target) > 0 || tcpSI.lastUsed < tsi.lastUsed))
                        {
                            lasttcpSI = tcpSI = tsi;
                            lastFound = target;
                        }
                }
            }
            if (tcpSI != null)
                using (new LockAndElevate(tcpSI.siLock))
                    tcpSI.lastUsed = Isis.NOW();
            return tcpSI;
        }

        internal static void CleanTCP()
        {
            List<TCPSockInfo> toClose = new List<TCPSockInfo>();
            using (new LockAndElevate(TCPLock))
            {
                List<Address> allDests = new List<Address>(), unNeeded = new List<Address>();
                foreach (TCPSockInfo tsi in TCPList)
                    if (tsi.initiatedByMe && !allDests.Contains(tsi.dest))
                        allDests.Add(tsi.dest);
                foreach (Address d in allDests)
                    if (Group.notNeeded(d))
                        unNeeded.Add(d);
                foreach (TCPSockInfo tsi in TCPList)
                    if (unNeeded.Contains(tsi.dest) && (Isis.NOW() - tsi.openedAt) > 10000)
                        toClose.Add(tsi);
            }
            foreach (TCPSockInfo tsi in toClose)
            {
                if ((IsisSystem.Debug & IsisSystem.TUNNELING) != 0)
                    Isis.WriteLine("CleanTCP[NOW=" + Isis.TimeToString(Isis.NOW()) + "]: removing an unneeded TCP connection to " + tsi.dest + " [" + TCPCnames[tsi.which] + "]");
                TCPClose(tsi, false);
            }
        }

        private static IPEndPoint AddressToIPEndpoint(Address dest, int which)
        {
            if (dest != null && !dest.isNull())
            {
                if (which == ACKBB)
                    return new IPEndPoint(dest.home, dest.ackPort);
                else if (which == RECVBB)
                    return new IPEndPoint(dest.home, dest.p2pPort);
            }
            return null;
        }

        private static int IPEndpointCompare(IPEndPoint A, IPEndPoint B)
        {
            int ah = A.GetHashCode(), bh = B.GetHashCode();
            Isis.WriteLine("IPEndpointCompare: A.hashcode = " + ah + ", B.hashcode=" + bh + ", portnos " + A.Port + " and " + B.Port);
            if (ah != bh)
                return ah - bh;
            return A.Port - B.Port;
        }

        internal static string TCPListState()
        {
            string s = "TCPListState:\r\n";
            using (new LockAndElevate(TCPLock))
                foreach (TCPSockInfo si in TCPList)
                {
                    using (new LockAndElevate(si.siLock))
                    {
                        string d = "";
                        if (si.dest != null)
                            d = si.dest.ToString();
                        else
                        {
                            byte[] b = si.remoteEndPoint.Address.GetAddressBytes();
                            d = b[0] + "." + b[1] + "." + b[2] + "." + b[3] + "::" + si.remoteEndPoint.Port;
                        }
                        s += "  TCPList[" + si.index + "; dest=" + d + "]{ByMe=" + si.initiatedByMe + ", " + (si.closed ? "closed" : "open") +
                            ", type=" + TCPCnames[si.which] + ", lastUsed=" + Isis.TimeToString(si.lastUsed) +
                            ", socket=" + (si.theSocket == null ? "null" : si.theSocket.Handle.ToString()) +
                            "}, nwrites=" + si.nwrites + ", nreads=" + si.nreads + "\r\n";
                    }
                }
            using (new LockAndElevate(ConnectingLock))
                foreach (AWPair awp in ConnectingTo)
                    s += "  TCP overlay is connecting to " + awp.dest + "::" + awp.which + ", pending send list has " + awp.toSend.Count() + " items\r\n";
            return s;
        }

        internal static void TCPNoteVDS(Isis.ViewDelta[] vds)
        {
            if (vds.Length == 0)
                return;
            int lastvid = vds[vds.Length - 1].prevVid;
            List<TCPSockInfo> toSend = new List<TCPSockInfo>();
            using (new LockAndElevate(TCPLock))
                foreach (TCPSockInfo si in TCPList)
                    if (si.lastISISMEMBERSViewDelta < lastvid)
                    {
                        si.lastISISMEMBERSViewDelta = lastvid;
                        toSend.Add(si);
                    }
            foreach (TCPSockInfo si in toSend)
            {
                byte[] ba = Msg.toBArray(vds);
                TCPhdr tcphdr = new TCPhdr(TCPC_VDS, ba.Length);
                using (new LockAndElevate(si.siLock))
                    ++si.nwrites;
                byte[] ha = Msg.toBArray(tcphdr);
                SendAllBytes(si, ha, ba);
            }
        }

        internal static void TCPShutdown()
        {
            List<TCPSockInfo> copy = new List<TCPSockInfo>();
            using (new LockAndElevate(TCPLock))
                foreach (TCPSockInfo si in TCPList)
                    copy.Add(si);
            foreach (TCPSockInfo si in copy)
                si.TCPBB.put(null);
        }

        private static void callDispose(TCPSockInfo si)
        {
#if !__MonoCS__
            si.theSocket.Dispose();
#endif // !__MonoCS__
        }

        internal static void TCPListCheck()
        {
            List<TCPSockInfo> toDelete = new List<TCPSockInfo>();
            using (new LockAndElevate(TCPLock))
            {
                foreach (TCPSockInfo tcpsi in TCPList)
                {
                    using (new LockAndElevate(tcpsi.siLock))
                    {
                        bool isDup = false;
                        foreach (TCPSockInfo si in TCPList)
                            if (si == tcpsi)
                                break;
                            else if (si.dest.Equals(tcpsi.dest) && si.which == tcpsi.which)
                                isDup = true;
                        if (!isDup)
                            continue;
                    }
                    toDelete.Add(tcpsi);
                }
            }
            foreach (TCPSockInfo tcpsi in toDelete)
                TCPClose(tcpsi, false);
        }

        internal static bool TCPSendIfKnown(Address who, byte[] data, int which)
        {
            if (!Isis.ISIS_TCP_ONLY)
                return false;
            if (ConnectingToCheck(who, data, which))
                return true;
            if (FindTPCsi(who, null, which) != null)
            {
                TCPSendTo(who, data, null, which);
                return true;
            }
            return false;
        }

        internal static void TCPConnectAsWorker(Address dest)
        {
            TCPConnectTo(dest, new IPEndPoint(dest.home, Isis.ISIS_DEFAULT_PORTNOa), SEEKU, true, null);
        }

        // This routine tends to get called simultaneously with the same who/which pairs and I've had some difficulty
        // getting it to behave itself (e.g. make the connection for just one of them)
        internal static Socket TCPConnectTo(Address dest, IPEndPoint remoteEP, int which, bool needLock, byte[] buf)
        {
            TCPSockInfo si;
            if (!Isis.ISIS_TCP_ONLY)
                throw new IsisException("TCPConnectTo but not in ISIS_TCP_ONLY mode!");
            if (dest.isMyAddress() || ConnectingToCheck(dest, buf, which, true))
                return null;
            if ((si = FindTPCsi(dest, remoteEP, which)) != null)
            {
                si.initiatedByMe = true;
                if (buf != null)
                    TCPSendTo(dest, buf, remoteEP, which);
                return null;
            }
            Socket s = null;
            try
            {
                s = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
                TCPOverlaySetupSocket(s);
                IAsyncResult result = s.BeginConnect((EndPoint)remoteEP, null, null);
                result.AsyncWaitHandle.WaitOne(30000, true);
                if (!s.Connected)
                {
                    if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                        Isis.WriteLine("Close si.theSocket=" + s.Handle + " (TCP ConnectTo !s.Connected)");
                    s.Close();
                    callDispose(s);
                    return null;
                }
                if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                    Isis.WriteLine("TCPConnectTo: Initiating<" + TCPCnames[which] + "> connection from " + Isis.my_address + " to " + dest + " at endpoint " + remoteEP);
                int bsa = s.Send(Msg.toBArray(new TCPhdr((which == ACKBB) ? TCPC_ACKCH : (which == RECVBB ? TCPC_P2PCH : TCPC_SEEKU), Isis.my_address_bytes.Length)));
                int bsb = s.Send(Isis.my_address_bytes);
                using (new LockAndElevate(IsisSystem.RTS.Lock))
                {
                    IsisSystem.RTS.TCPBsent += bsa;
                    IsisSystem.RTS.TCPsent++;
                    IsisSystem.RTS.TCPBsent += bsb;
                    IsisSystem.RTS.TCPsent++;
                }
                if (which == SEEKU)
                {
                    s.Close();
                    callDispose(s);
                    return null;
                }
            }
            catch
            {
                if (s != null)
                {
                    s.Close();
                    callDispose(s);
                }
                return null;
            }
            if (!dest.isNull() && !dest.isMyAddress())
                new Thread(delegate()
                {
                    try
                    {
                        Thread.CurrentThread.Name = "ConnectTo socket for " + dest + ": mode = " + TCPCnames[which];
                        TCPSockInfo TCPsi = new TCPSockInfo(dest, remoteEP, s, which, true);
                        TCPReceiveOn(TCPsi, (which == ACKBB) ? AckBB : RecvBB, dest, which);
                        terminate(TCPsi);
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    IsisSystem.ThreadTerminationMagic();
                }).Start();
            return s;
        }

        private static void callDispose(Socket s)
        {
#if !__MonoCS__
            s.Dispose();
#endif // !__MonoCS__
        }

        internal static void TCPListen()
        {
            if (!Isis.ISIS_TCP_ONLY)
                throw new IsisException("TCPListen but not in ISIS_TCP_ONLY mode!");
            TCPListenOn(my_acksocket, "acksocket", ACKBB);
            TCPListenOn(my_p2psocket, "p2psocket", RECVBB);
        }

        internal static void TCPListenOn(Socket s, string ws, int which)
        {
            new Thread(delegate()
            {
                Thread.CurrentThread.Name = "TCP listener (" + ws + ")";
                try
                {
                    s.Listen(32);
                    Socket theSocket;
                    while ((theSocket = s.Accept()) != null)
                    {
                        if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                            Isis.WriteLine("Accepted a new connection... placing it into NoDelay mode and reading header information in a new thread ");
                        Socket newSocket = theSocket;
                        new Thread(delegate()
                        {
                            TCPSockInfo newConnect = null;
                            try
                            {
                                Address dest;
                                TCPhdr tcphdr;
                                TCPOverlaySetupSocket(newSocket);
                                newConnect = new TCPSockInfo(null, (IPEndPoint)newSocket.RemoteEndPoint, newSocket, which, false);
                                if (TCPHdrLen == 0)
                                    TCPHdrLen = Msg.toBArray(new TCPhdr()).Length;
                                byte[] hdr = new byte[TCPHdrLen];
                                using (new LockAndElevate(newConnect.rcvLock))
                                {
                                    if (!RecvFixedlen(newConnect, TCPHdrLen, hdr))
                                    {
                                        terminate(newConnect);
                                        return;
                                    }
                                    tcphdr = (TCPhdr)Msg.BArrayToObjects(hdr)[0];
                                    if (tcphdr.len != Isis.my_address_bytes.Length)
                                    {
                                        terminate(newConnect);
                                        Isis.WriteLine("WARNING: Discarding an incoming packet because the TCP header length was " + tcphdr.len + " instead of " + Isis.my_address_bytes.Length);
                                        return;
                                    }
                                    byte[] adr = new byte[tcphdr.len];
                                    if (!RecvFixedlen(newConnect, tcphdr.len, adr))
                                    {
                                        terminate(newConnect);
                                        return;
                                    }
                                    dest = (Address)Msg.BArrayToObjects(adr)[0];
                                }
                                // Now that we know the true address, switch to using it.  The remoteEndPoint was only a temporary "fix"
                                if (dest.isMyAddress())
                                {
                                    terminate(newConnect);
                                    return;
                                }
                                newConnect.dest = dest;
                                newConnect.remoteEndPoint = AddressToIPEndpoint(dest, which);
                                newConnect.TCPBB.name += dest;
                                Thread.CurrentThread.Name = "TCP receiver for " + dest + ": mode=" + TCPCommands[tcphdr.TCPCommand];
                                if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                                    Isis.WriteLine("My new connection is from " + dest + ", command=" + TCPCommands[tcphdr.TCPCommand]);
                                switch (tcphdr.TCPCommand)
                                {
                                    case TCPC_ACKCH:
                                    case TCPC_P2PCH:
                                        TCPReceiveOn(newConnect, (tcphdr.TCPCommand == TCPC_P2PCH) ? RecvBB : AckBB, dest, tcphdr.TCPCommand);
                                        terminate(newConnect);
                                        return;

                                    case TCPC_SEEKU:
                                        terminate(newConnect);
                                        if (FindTPCsi(dest, null, which) != null)
                                            return;
                                        TCPConnectTo(dest, new IPEndPoint(dest.home, dest.p2pPort), RECVBB, true, null);
                                        TCPConnectTo(dest, new IPEndPoint(dest.home, dest.ackPort), ACKBB, true, null);
                                        if (Isis.ORACLE.IAmLeader())
                                        {
                                            if ((IsisSystem.Debug & IsisSystem.STARTSEQ) != 0)
                                                Isis.WriteLine("Received a SEEKU inquiry in ORACLE, sending a message to inhibit creation of new ORACLEs to " + dest);
                                            Isis.ORACLE.doPureP2PSend(dest, true, Isis.JOIN, Isis.my_address);
                                        }
                                        return;

                                    default:
                                        terminate(newConnect);
                                        return;
                                }
                            }
                            catch (Exception e)
                            {
                                Isis.WriteLine("Exception: " + e);
                                terminate(newConnect);
                                return;
                            }
                        }).Start();
                    }
                }
                catch { IsisSystem.Shutdown("IsisSystem.Shutdown from TCPListenOn: " + ws); }
                IsisSystem.ThreadTerminationMagic();
            }).Start();
        }

        private static void terminate(TCPSockInfo newConnect)
        {
            newConnect.TCPBB.put(null);
            try { newConnect.theSocket.Close(); }
            catch { }
            callDispose(newConnect.theSocket);
        }

        internal static void TCPReceiveOn(TCPSockInfo si, BoundedBuffer bb, Address who, int which)
        {
            if (!Isis.ISIS_TCP_ONLY)
                throw new IsisException("TCPReceiveOn but not in ISIS_TCP_ONLY mode!");
            if (who.isMyAddress())
            {
                si.TCPBB.put(null);
                return;
            }
            using (new LockAndElevate(si.rcvLock))
            {
                if (si.TCPReceiver != null)
                {
                    si.TCPBB.put(null);
                    return;
                }
                si.TCPReceiver = Thread.CurrentThread;
            }
            if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                Isis.WriteLine("TCPReceiveOn: " + who + ", " + TCPCnames[which] + "... entry with " + TCPListState());
            if (si.dest == null)
                si.dest = who;
            if (si.which != which)
                throw new IsisException("Confused in TCPReceiveOn");
            bool fnd = false;
            using (new LockAndElevate(TCPLock))
            {
                if (si.closed)
                    return;
                foreach (TCPSockInfo tcpsi in TCPList)
                    if (tcpsi.dest.Equals(si.dest) && tcpsi.which == which)
                    {
                        fnd = true;
                        break;
                    }
                if (!fnd)
                    TCPList.Add(si);
            }
            if (fnd)
            {
                TCPClose(si, true);
                return;
            }
            ConnectingToDone(si);
            while (IsisSystem.IsisActive)
            {
                IsisSystem.RTS.ThreadCntrs[19]++;
                try
                {
                    if (TCPHdrLen == 0)
                        TCPHdrLen = Msg.toBArray(new TCPhdr()).Length;
                    if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                        Isis.WriteLine("RECEIVING ON :" + si.remoteEndPoint);
                    byte[] hdr = new byte[TCPHdrLen];
                    using (new LockAndElevate(si.rcvLock))
                    {
                        if (!RecvFixedlen(si, TCPHdrLen, hdr))
                        {
                            if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                                Isis.WriteLine("Connection disrupted (channel broke) when trying to read TCP channel header, closing it");
                            TCPClose(si, false);
                            break;
                        }
                        object[] hdrAsObjs = Msg.BArrayToObjects(hdr);
                        if (hdrAsObjs == null || hdrAsObjs.Length == 0 || !hdrAsObjs[0].GetType().Equals(typeof(TCPhdr)))
                        {
                            if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                                Isis.WriteLine("Connection disrupted (damaged message) when trying to read TCP channel header, closing it");
                            TCPClose(si, false);
                            break;
                        }
                        TCPhdr tcphdr = (TCPhdr)hdrAsObjs[0];
                        if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                            Isis.WriteLine("RECEIVED HEADER from " + si.dest + ", command=" + TCPCommands[tcphdr.TCPCommand] + ", len=" + tcphdr.len);
                        switch (tcphdr.TCPCommand)
                        {
                            case TCPC_ACKCH:
                            case TCPC_P2PCH:
                                {
                                    if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                                        Isis.WriteLine("Connection disrupted (unexpected ackch/p2pch) when trying to read TCP channel header, closing it");
                                    TCPClose(si, true);
                                    return;
                                }

                            case TCPC_MSG:
                                byte[] buf = new byte[tcphdr.len];
                                using (new LockAndElevate(si.siLock))
                                    ++si.nreads;
                                if (RecvFixedlen(si, tcphdr.len, buf))
                                {
                                    if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                                        Isis.WriteLine("TCPRcv<socket=" + si.theSocket.Handle + "; " + TCPCnames[which] + "> remote partner= " + si.dest + " received " + buf.Length);
                                    bb.put(buf);
                                }
                                break;

                            case TCPC_CLOSE:
                                TCPClose(si, true);
                                return;

                            case TCPC_VDS:
                                using (new LockAndElevate(si.siLock))
                                    ++si.nreads;
                                byte[] vdba = new byte[tcphdr.len];
                                if (RecvFixedlen(si, tcphdr.len, vdba))
                                {
                                    object[] obs = Msg.BArrayToObjects(vdba, typeof(Isis.ViewDelta[]));
                                    if (Isis.ISISMEMBERS != null && obs != null && obs.Length == 1)
                                        Isis.ISISMEMBERS.gotNewViewDeltas((Isis.ViewDelta[])obs[0]);
                                }
                                break;
                        }
                    }
                }
                catch (Exception e) { Isis.WriteLine("TCPReceiveOn: " + e); return; }
            }
        }

        internal static void TCPClose(TCPSockInfo si, bool gotClose)
        {
            if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                Isis.WriteLine("TCPClose for " + (si.dest == null ? "null-channel id" : si.dest.ToString()) + ", gotClose=" + gotClose);
            using (new LockAndElevate(TCPLock))
            {
                if (si.closed)  // True after the socket reader reads a null from the BB
                    return;
                if (TCPList.Contains(si))
                    TCPList.Remove(si);
            }
            if (!gotClose)
            {
                TCPhdr CloseIt = new TCPhdr(TCPC_CLOSE, 0);
                si.TCPBB.put(Msg.toBArray(CloseIt));
            }
            si.TCPBB.put(null);
            List<TCPSockInfo> toClose = new List<TCPSockInfo>();
            using (new LockAndElevate(TCPLock))
                foreach (TCPSockInfo osi in TCPList)
                    if (osi.dest.Equals(si.dest) && osi.which != si.which && osi.initiatedByMe == si.initiatedByMe)
                        // If I closed one of the two connections, close the other as well
                        toClose.Add(osi);
            if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                Isis.WriteLine("TCPClose finished, new state: " + TCPListState());
            foreach (TCPSockInfo tc in toClose)
                TCPClose(tc, false);
        }

        // Call with a lock on si.rcvLock
        private static bool RecvFixedlen(TCPSockInfo si, int len, byte[] buf)
        {
            int tryLimit = 25;
            int idx = 0;
            while (len > 0)
            {
                int nb;
                try
                {
                    nb = si.theSocket.Receive(buf, idx, len, SocketFlags.None);
                    using (new LockAndElevate(IsisSystem.RTS.Lock))
                    {
                        IsisSystem.RTS.TCPBrcvd += nb;
                        IsisSystem.RTS.TCPrcvd++;
                    }
                }
                catch (SocketException e)
                {
                    if (e.SocketErrorCode != SocketError.WouldBlock || --tryLimit == 0)
                        return false;
                    nb = 0;
                    Thread.Sleep(25);
                }
                catch
                {
                    return false;
                }
                idx += nb;
                len -= nb;
            }
            using (new LockAndElevate(si.siLock))
                si.lastUsed = Isis.NOW();
            return len == 0;
        }

        static bool tokenThreadActive = false;

        internal static void resetTheToken(Group g)
        {
            if ((g.flags & Group.G_ISLARGE) != 0)
            {
                Group.tokenInfo theToken;
                using (new LockAndElevate(g.TokenLock))
                {
                    Group.tokenInfo.newToken(g);
                    theToken = g.theToken;
                }
                if (g == Isis.ISISMEMBERS && g.IAmRank0())
                    g.becomeGroupOwner();
                theToken.inhibitResenderLoop = true;
                theToken.gotAllAt = 0;
                theToken.logicalClock++;
                theToken.resetStableByLevel(g);
                using (new LockAndElevate(PendingSendBufferLock))
                    foreach (MsgDesc md in LgPendingSendBuffer)
                        if (md.group == g)
                        {
                            md.retryCnt = 0;
                            md.retryDelay = Math.Max(250, Math.Min(20, md.retryDelay));
                            md.remulticastTime = Isis.NOW() + md.retryDelay;
                        }
            }
        }

        internal class TThreadTuple
        {
            volatile internal Group.tokenInfo theToken;
            internal Group theGroup;
            internal int TheNextMsgID;

            internal TThreadTuple(Group.tokenInfo t, Group g, int n)
            {
                theToken = t; theGroup = g; TheNextMsgID = n;
            }
        }

        internal static long lastGCTime = 0;

        internal static void TokenThread()
        {
            try
            {
                beTokenThread();
            }
            catch (IsisShutdown)
            {
                return;
            }
            catch (Exception e)
            {
                if (!IsisSystem.IsisActive)
                    goto done;
                Isis.WriteLine("WARNING: Token thread exited abnormally with exception " + e);
                throw new IsisException("Dead token thread");
            }
        done:
            IsisSystem.ThreadTerminationMagic();
        }

        internal static void beTokenThread()
        {
            while (!IsisSystem.IsisActive)
                Thread.Sleep(250);
            try
            {
                if (tokenThreadActive)
                    return;
                tokenThreadActive = true;
                int loopCnt = 0;
                while (IsisSystem.IsisActive)
                {
                    IsisSystem.RTS.ThreadCntrs[20]++;
                    loopCnt++;
                    int tokenDelay = Isis.ISIS_TOKEN_DELAY;
                    if ((Isis.NOW() - lastGCTime) > Isis.ISIS_GCFREQ && FlowControl.MsgCountNum > 10)
                    {
                        lastGCTime = Isis.NOW();
                        GC.Collect();
                    }

                    bool fnd = false;

                    List<TThreadTuple> tokenList = new List<TThreadTuple>();

                    // minimizes lock hold time and also avoids a deadlock scenario...
                    List<Group> glist = Group.IsisGroupsClone();
                    foreach (Group g in glist)
                        if ((g.flags & Group.G_ISLARGE) == 0 || (loopCnt % 20 != 0 && !g.interesting))
                            continue;
                        else
                            using (new ILock(ILock.LLBRIEF, g.gaddr))
                            {
                                Group.tokenInfo theToken;
                                g.interesting = false;
                                using (new LockAndElevate(g.TokenLock))
                                    theToken = g.theToken;
                                if (theToken != null)
                                {
                                    g.updateSeenInfo();
                                    int nm = g.theView.NextIncomingMsgID[1];
                                    using (new LockAndElevate(g.ViewLock))
                                        if (g.nextMsgid != -1)
                                            nm = Math.Min(g.nextMsgid, g.theView.NextIncomingMsgID[1]);
                                    tokenList.Add(new TThreadTuple(theToken, g, nm));
                                }
                            }
                    foreach (TThreadTuple ttt in tokenList)
                    {
                        Group.tokenInfo theToken;
                        View theView;
                        using (new LockAndElevate(ttt.theGroup.TokenLock))
                        using (new LockAndElevate(ttt.theGroup.ViewLock))
                        {
                            theToken = ttt.theToken;
                            theView = ttt.theGroup.theView;
                            if (theToken.viewid < theView.viewid || theToken.WorkingView.viewid < theView.viewid)
                                // To maximize asynchrony, theView might sometimes be updated before WorkingView.  During such a period, don't send tokens
                                continue;
                            // Basically, we want to keep advancing stableTo to the maximum value confirmed by the other ring members, which would be 
                            // tp.StableByLevel[tp.next.Length-1] (the looped-back value for ring level n-1, if I'm at level n)
                            // Complications arise because during LgFlush, the value sent out is int.MaxValue, hence one or both of 
                            // tp.stableTo and tp.StableByLevel could be pegged at infinity.  So we need to trim that down to size.
                            // So: take the max; if a sensible number, done.  Else if it was infinity, take the min (assuming one of the two was
                            // pegged at infinity).  If no longer infinity, done.  Else just whatever my own next incoming msgID happens to be
                            if (theToken.IAmLgOwner)
                            {
                                if ((IsisSystem.Debug & IsisSystem.TOKENSTABILITY) != 0)
                                    Isis.WriteLine("In token thread I am the lgOwner for group <" + ttt.theGroup.gname + "> at time " + Isis.TimeToString(Isis.NOW()) + " (logicalClock=" + theToken.logicalClock + ") and stableTo = " + theToken.stableTo + " and tp.StableByLevel[tp.StableByLevel.Length=" + theToken.StableByLevel.Length + " - 1]=" +
                                        theToken.StableByLevel[theToken.StableByLevel.Length - 1] + " and members.length=" + theToken.WorkingView.members.Length + " and nextIncoming=" + (theView.NextIncomingMsgID[1] - 1));
                                int old = theToken.stableTo;
                                if (theToken.StableByLevel[theToken.StableByLevel.Length - 1] != int.MaxValue)
                                    theToken.stableTo = Math.Max(theToken.stableTo, theToken.StableByLevel[theToken.StableByLevel.Length - 1]);
                                else if (theToken.WorkingView.members.Length == 1)
                                    theToken.stableTo = theView.NextIncomingMsgID[1] - 1;
                                if ((IsisSystem.Debug & IsisSystem.TOKENSTABILITY) != 0 && theToken.stableTo != old)
                                    Isis.WriteLine("<" + ttt.theGroup.gname + ">: Decided to set stableTo to " + theToken.stableTo + " (was " + old + ") in <" + ttt.theGroup.gname +
                                        ">; this is because tp.StableByLevel[" + (theToken.next.Length - 1) + "]=" + theToken.StableByLevel[theToken.next.Length - 1]);
                            }
                        }
                        if (ttt.theGroup.gcollectedTo < theToken.stableTo)
                        {
                            ttt.theGroup.gcollectedTo = theToken.stableTo;
                            if ((IsisSystem.Debug & IsisSystem.TOKENLAYER) != 0)
                                Isis.WriteLine("LgOwner: LgGCollect to: " + ttt.theGroup.gcollectedTo);
                            ReliableSender.gotLgAck(ttt.theGroup, theToken.stableTo);
                        }
                        for (int slevel = 0; slevel < theToken.next.Length; slevel++)
                        {
                            using (new LockAndElevate(theToken.tokenInMotionLock))
                                if (theToken.tokenInMotion[slevel] > 2)
                                    // Only two can be pending at a time to any given (target,level) pair...
                                    continue;

                            if ((IsisSystem.Debug & IsisSystem.TOKENSTABILITY) != 0)
                                Isis.WriteLine("In token thread deciding to send a token in group <" + ttt.theGroup.gname + "> at time " + Isis.TimeToString(Isis.NOW()));
                            Address nextInRing = theToken.next[slevel];
                            theToken.tokenLevel = slevel;
                            if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                                Isis.WriteLine("In TokenThread computing tp.token for level " + slevel + ", sv=" + theToken.stableAtSender + ", nextInRing " + ((nextInRing == null) ? "NULL" : nextInRing.ToString()));

                            if (nextInRing == null || nextInRing.isMyAddress())
                            {
                                // Special case for a degenerate ring.  Might not be degenerate at some higher level
                                if ((IsisSystem.Debug & IsisSystem.TOKENSTABILITY) != 0)
                                    Isis.WriteLine("Stability logic[slevel=" + slevel + "]: TokenThread encountered special case for a degenerate ring (setting StableByLevel[" + slevel + "] by copying from below)");
                                theToken.StableByLevel[slevel] = (slevel == 0) ? theToken.stableAtSender : theToken.StableByLevel[slevel - 1];
                                if ((IsisSystem.Debug & IsisSystem.AGGREGATION) != 0)
                                    Isis.WriteLine("Aggregation logic[slevel=" + slevel + "]: TokenThread encountered special case for a degenerate ring (copying DValues up to next level)");
                                if (slevel < theToken.theGroup.AggList.Length - 1)
                                {
                                    int idx = 0;
                                    foreach (IAggregateEventHandler ae in theToken.theGroup.AggList[slevel])
                                    {
                                        int ndx = 0;
                                        foreach (IAggregateEventHandler nlae in theToken.theGroup.AggList[slevel + 1])
                                            if (idx == ndx)
                                            {
                                                // Copies DValues from level k to level k+1
                                                nlae.LoadDValues(ae.GetDValues());
                                                break;
                                            }
                                            else
                                                ++ndx;
                                        ++idx;
                                    }
                                }
                                continue;
                            }

                            if (theToken.myOffset[slevel] != 0)
                            {
                                // An inner node in some ring: outgoing value of aggStable is the smaller of last value received in a valid token
                                // and my own value.  for level 0, this simply reflects my own delivered messages queue.  For levels>0, this will be the stability aggregated
                                // at the next level down.  The token will also contain my value of myStable but that's just to let the next guy in the ring do NACK-triggered retransmissions
                                int localStabilityValue = theToken.stableAtSender;
                                if (slevel > 0)
                                    localStabilityValue = Math.Min(localStabilityValue, theToken.StableByLevel[slevel - 1]);
                                if ((IsisSystem.Debug & IsisSystem.TOKENSTABILITY) != 0)
                                    Isis.WriteLine("<" + ttt.theGroup.gname + ">: Setting tp.aggStable = Math.Min(tp.StableByLevel[" + slevel + "]=" + theToken.StableByLevel[slevel] + ", localStabilityValue=" + localStabilityValue + ")");
                                theToken.aggStable = Math.Min(theToken.StableByLevel[slevel], theToken.stableAtSender);
                            }
                            else
                            {
                                // The "point man" in some ring.  At level zero you use your own value if on the left since there is no input from the left
                                if (slevel == 0)
                                {
                                    theToken.aggStable = theToken.stableAtSender;
                                    if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENSTABILITY)) != 0)
                                        Isis.WriteLine("<" + ttt.theGroup.gname + ">: ring leader logic: I " + (theToken.IAmLgOwner ? "am" : "am not") +
                                            " the group owner, setting theToken.aggStable to " + theToken.aggStable);
                                }
                                else
                                {
                                    // Higher levels use the aggregated value from lower ones as their "starting proposal".  This is retained from the last valid token received down there
                                    theToken.aggStable = theToken.StableByLevel[slevel - 1];
                                    if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.TOKENSTABILITY)) != 0)
                                        Isis.WriteLine("<" + ttt.theGroup.gname + ">: ring leader logic: I " + (theToken.IAmLgOwner ? "am" : "am not") +
                                            " the group owner, setting theToken.aggStable to triple.theToken.StableByLevel[" + (slevel - 1) + "]= " + theToken.aggStable);
                                }

                            }

                            if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                                Isis.WriteLine("Sending tp.token with callback requested for level " + slevel + " to " + nextInRing + ", value " + theToken);

                            using (new LockAndElevate(theToken.tokenInMotionLock))
                                theToken.tokenInMotion[slevel]++;

                            if (!nextInRing.isMyAddress())
                            {
                                int mylevel = slevel;
                                SendP2P(Msg.ISTOKEN, nextInRing, ttt.theGroup, theToken.toBArray(), true, delegate()
                                {
                                    if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                                        Isis.WriteLine("Token-sent callback for group <" + theToken.theGroup.gname + ">, level " + mylevel + " (legal range: 0.." + theToken.tokenInMotion.Length + ")");
                                    using (new LockAndElevate(theToken.tokenInMotionLock))
                                        theToken.tokenInMotion[mylevel]--;
                                });
                                using (new LockAndElevate(IsisSystem.RTS.Lock))
                                    IsisSystem.RTS.TokensSent++;
                            }
                            if (ttt.theToken != theToken)
                                continue;
                            if (theToken.lastToken[slevel] == null || !theToken.lastToken[slevel].sender.Equals(theToken.last[slevel]))
                            {
                                if (!theToken.pinged[slevel])
                                    pingLast(theToken, ttt, slevel);
                                theToken.pinged[slevel] = true;
                            }
                            else
                                theToken.pinged[slevel] = false;
                        }
                    }
                    if (fnd)
                        tokenDelay >>= 2;
                    if (PendingSendBuffer.Count() > Isis.ISIS_ASYNCMTOTALLIMIT)
                        tokenDelay = Math.Max(tokenDelay, 250);
                    Thread.Sleep(Math.Max(2, tokenDelay));
                }
            }
            catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
        }

        private static void pingLast(Group.tokenInfo tp, TThreadTuple tokenSummary, int slevel)
        {
            if (!tp.sentAToken[slevel] && !tp.last[slevel].isMyAddress())
            {
                tp.sentAToken[slevel] = true;
                // This will be treated as an "invalid" token (I'm not the guy he expected tokens from) but the view deltas piggybacked on it still get processed
                SendP2P(Msg.ISTOKEN, tp.last[slevel], tokenSummary.theGroup, tp.toBArray(), true);
            }
        }

        internal static void gotToken(Msg m)
        {
            Group g;
            using (new LockAndElevate(IsisSystem.RTS.Lock))
                IsisSystem.RTS.TokensRcvd++;
            if ((m.flags & Msg.FRAGGED) != 0)
            {
                // This code cheats, stashing the gaddr in the token dest field because m.gaddr isn't part of the wire representation of a message
                // This won't confuse the message ordering layers because tokens are handled by special logic that doesn't worry about sequencing
                g = Group.Lookup(m.dest);
                if (g == null)
                {
                    if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                        Isis.WriteLine("WARNING: Discarding a fragmented token that references group " + m.dest + " because I'm not a member of that group");
                    return;
                }
                m.flags |= Msg.TOKEN;
                Group.deFragRdv(g, m);
                return;
            }
            Group.tokenInfo toke = new Group.tokenInfo(m.payload);
            toke.whenReceived = Isis.NOW();
            g = toke.theGroup;
            Group.tokenInfo theToken = null;
            if (g != null)
                using (new LockAndElevate(g.TokenLock))
                    theToken = g.theToken;
            if (g == null || theToken == null)
            {
                if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.DISCARDS)) != 0)
                    Isis.WriteLine("gotToken: I'm not in group " + toke.gaddr + "(or it has no token)!");
                using (new LockAndElevate(IsisSystem.RTS.Lock))
                    IsisSystem.RTS.Discarded++;
                return;
            }
            if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                Isis.WriteLine("gotToken: call processIncoming: " + toke);
            g.processIncoming(m.sender, toke);
        }

        internal static void NodeHasFailed(Address who)
        {
            List<MsgDesc> toRemove = new List<MsgDesc>();
            using (new LockAndElevate(PendingSendBufferLock))
            {
                LinkedList<MsgDesc> newPSB = new LinkedList<MsgDesc>();
                foreach (MsgDesc md in PendingSendBuffer)
                {
                    if (!md.dest.Equals(who))
                        newPSB.AddLast(md);
                    else
                    {
                        if (md.group != null)
                        {
                            bool wantsStabilitySent;
                            using (new LockAndElevate(md.group.GroupFlagsLock))
                                wantsStabilitySent = md.group.SendStabilityNeeded;
                            if (md.localSender && !wantsStabilitySent && md.type != Msg.STABILITYINFO)
                            {
                                wantsStabilitySent = md.group.multicastSendIsStable(md.MsgVid, md.MsgId);
                                using (new LockAndElevate(md.group.GroupFlagsLock))
                                    md.group.SendStabilityNeeded = wantsStabilitySent;
                            }
                        }
                        toRemove.Add(md);
                    }
                }
                PendingSendBuffer = newPSB;
            }
            int cnt = toRemove.Count();
            foreach (MsgDesc md in toRemove)
            {
                md.done = true;
                FlushingRemove(md, "NodeHasFailed", --cnt == 0);
            }
            AwaitReplies.doNoteFailure(who);
            ReliableSender.P2PSequencer.Failed(who);
            IsisSystem.WriteAckInfo();
        }

        internal static void gotLgAck(Group g, int low)
        {
            if ((IsisSystem.Debug & IsisSystem.TOKENLOGIC) != 0)
                Isis.WriteLine("->->  in gotLgAck group " + g.gname + ", g.lastLgAckID=" + g.lastLgAckID + ", low=" + low);
            if (low < 0 || g.lastLgAckID == low)
                return;
            g.lastLgAckID = low;
            List<ReliableSender.MsgDesc> toRemove = new List<MsgDesc>();
            using (new LockAndElevate(PendingSendBufferLock))
            {
                LinkedList<MsgDesc> newLgPendingSendBuffer = new LinkedList<MsgDesc>();
                foreach (MsgDesc lgmd in LgPendingSendBuffer)
                    if (lgmd.dest.Equals(g.gaddr) && lgmd.UID <= low)
                        toRemove.Add(lgmd);
                    else
                        newLgPendingSendBuffer.AddLast(lgmd);
                LgPendingSendBuffer = newLgPendingSendBuffer;
            }
            int cnt = toRemove.Count();
            foreach (ReliableSender.MsgDesc lgmd in toRemove)
            {
                lgmd.done = true;
                FlushingRemove(lgmd, "lgmd.gotLgAck", --cnt == 0);
            }
        }

        internal static void FlushingRemove(MsgDesc md, string where, bool checkBarrier)
        {
            List<Semaphore> theCPSList;
            lock (md)
            {
                if (md.alreadyCleanedUp)
                    return;
                md.alreadyCleanedUp = true;
                theCPSList = md.CPSList;
                md.CPSList = null;
                if (md.group != null)
                    md.group.interesting = true;
            }
            FlowControl.FCEndSend(md.num, md.denom);
            if (theCPSList != null)
                lock (theCPSList)
                    foreach (Semaphore cps in theCPSList)
                        cps.Release(1);
            if (checkBarrier)
                using (new LockAndElevate(ReliableSender.rWaitLock))
                    if (ReliableSender.rWaiting > 0 && RemoteBacklogCount() < Isis.ISIS_MAXRBACKLOG * 7 / 10)
                        ILock.Barrier(ILock.LLWAIT, ILock.LFLOWCNTRLB).BarrierReleaseAll();
            if (md.am != null)
            {
                using (new LockAndElevate(activeMsgLock))
                {
                    md.am.StableCnt++;
                    List<msgWaiter> newList = new List<msgWaiter>();
                    if (md.am.Waiters != null)
                    {
                        foreach (msgWaiter mw in md.am.Waiters)
                            if (md.am.StableCnt >= mw.threshold)
                                mw.waitSema.Release(1);
                            else
                                newList.Add(mw);
                        if (newList.Count() > 0)
                            md.am.Waiters = newList;
                        else
                            md.am.Waiters = null;
                    }
                    if (md.am.StableCnt == md.am.SentCnt)
                        if (md.am.Waiters != null)
                            throw new IsisException("About to remove activeMsg object yet wait queue is non-null!");
                        else
                            activeMsgs.Remove(md.am);
                }
            }
            if (md.p2pdcb != null)
                md.p2pdcb();
        }

        internal static void waitForStability(int threshold)
        {
            List<Semaphore> slist = new List<Semaphore>();
            using (new LockAndElevate(activeMsgLock))
                foreach (activeMsg am in activeMsgs)
                    if (am.StableCnt < Math.Min(threshold, am.SentCnt))
                    {
                        Semaphore s = new Semaphore(0, int.MaxValue);
                        if (am.Waiters == null)
                            am.Waiters = new List<msgWaiter>();
                        am.Waiters.Add(new msgWaiter(Math.Min(threshold, am.SentCnt), s));
                        slist.Add(s);
                    }
            foreach (Semaphore s in slist)
                s.WaitOne();
        }

        internal class P2PSequencer
        {
            internal delegate void P2PCB(Msg m);

            internal class P2PCBList
            {
                internal Msg theMsg;
                internal P2PCB theCB;

                internal P2PCBList(Msg m, P2PCB cb) { theMsg = m; theCB = cb; }
            }

            internal Address dest;
            internal int outSeqn;
            internal int inSeqn;
            internal int remoteBacklog;
            internal LockObject Lock = new LockObject("P2PSequencer.Lock");
            internal SortedList<int, P2PCBList> outOfOrder = new SortedList<int, P2PCBList>();
            internal List<KeyValuePair<int, P2PCBList>> callbacksToDo = new List<KeyValuePair<int, P2PCBList>>();
            internal static Dictionary<Address, P2PSequencer> PSIndex = new Dictionary<Address, P2PSequencer>(1000);
            internal static LockObject PSListLock = new LockObject("PSListLock", ThreadPriority.Highest);
            internal long lastCallbackTime;

            internal P2PSequencer(Address who)
            {
                dest = who;
            }

            internal string GetP2PSState()
            {
                string s = "-- P2P Sequencer state for OutOfOrder queue of " + dest + ", inseqn " + inSeqn + ", outseqn " + outSeqn +
                    ", last callback was at time " + Isis.TimeToString(lastCallbackTime) + ", remote backlog=" + getRBacklog(this) + "\r\n";
                foreach (KeyValuePair<int, P2PCBList> kvp in outOfOrder)
                    s += "  **  " + kvp.Value.theMsg + "\r\n";
                if (callbacksToDo.Count() > 0)
                {
                    s += "  ++ Doing a p2p callback, when it finishes will deliver: \r\n";
                    foreach (KeyValuePair<int, P2PCBList> kvp in callbacksToDo)
                        s += "  **  " + kvp.Value.theMsg + "\r\n";
                }
                return s;
            }

            internal static string GetState()
            {
                using (new LockAndElevate(PSListLock))
                {
                    if (PSIndex.Count() == 0)
                        return "";
                    string s = "P2PSequencer states\r\n";
                    foreach (var kvp in PSIndex)
                        using (new LockAndElevate(kvp.Value.Lock))
                            s += kvp.Value.GetP2PSState();
                    return s;
                }
            }

            internal static void Failed(Address who)
            {
                using (new LockAndElevate(PSListLock))
                {
                    P2PSequencer p2ps;
                    if (P2PSequencer.PSIndex.TryGetValue(who, out p2ps))
                        using (new LockAndElevate(p2ps.Lock))
                            PSIndex.Remove(p2ps.dest);
                }
            }

            internal static int NextP2PSeqn(string why, Address who)
            {
                using (new LockAndElevate(Isis.RIPLock))
                    if (Isis.RIPList.Contains(who))
                        return -1;
                using (new LockAndElevate(PSListLock))
                {
                    P2PSequencer p2ps;
                    if (!PSIndex.TryGetValue(who, out p2ps))
                        PSIndex.Add(who, p2ps = new P2PSequencer(who));
                    using (new LockAndElevate(p2ps.Lock))
                    {
                        if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.P2PLAYER)) != 0)
                            Isis.WriteLine("NextP2PSeqn<" + why + "> dest " + who + ", using P2P seqn " + p2ps.outSeqn);
                        return p2ps.outSeqn++;
                    }
                }
            }

            internal static void CheckP2PSeqn()
            {
                using (new LockAndElevate(PSListLock))
                    foreach (var kvp in PSIndex)
                        using (new LockAndElevate(kvp.Value.Lock))
                            if (kvp.Value.outOfOrder.Count() > 0 && (Isis.NOW() - kvp.Value.lastCallbackTime) > Isis.ISIS_DEFAULTTIMEOUT * 6 && !HeardFromRecently(kvp.Value.dest))
                                throw new IsisException("Message(s) trapped on a p2ps out-of-order or delivery queue for " + kvp.Value.dest);
            }

            internal static int CountP2PSeqn()
            {
                int cnt = 0;
                using (new LockAndElevate(PSListLock))
                    foreach (var kvp in PSIndex)
                        using (new LockAndElevate(kvp.Value.Lock))
                            cnt += kvp.Value.outOfOrder.Count() + kvp.Value.callbacksToDo.Count();
                return cnt;
            }

            internal static LockObject CBWRLock = new LockObject("CBWRLock");
            internal static bool Active = false;
            internal static LockObject P2PCBLock = new LockObject("P2PCBLock", ThreadPriority.AboveNormal);


            internal static void P2PCBWhenReady(Address sender, Msg m, P2PCB cb)
            {
                using (new LockAndElevate(Isis.RIPLock))
                    if (Isis.RIPList.Contains(sender))
                    {
                        if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                            Isis.WriteLine("Discarding a message in P2PCBWhenReady: sender " + sender + " is on RIP list");
                        using (new LockAndElevate(IsisSystem.RTS.Lock))
                            IsisSystem.RTS.Discarded++;
                        return;
                    }
                using (new LockAndElevate(P2PCBLock))
                {
                    if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.P2PLAYER)) != 0)
                        Isis.WriteLine("CBWhenReady called for " + m);
                    P2PSequencer p2ps = null;
                    using (new LockAndElevate(PSListLock))
                        if (!P2PSequencer.PSIndex.TryGetValue(sender, out p2ps))
                            PSIndex.Add(sender, p2ps = new P2PSequencer(sender));
                    using (new LockAndElevate(p2ps.Lock))
                    {
                        if (m.msgid < p2ps.inSeqn)
                        {
                            if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.P2PLAYER | IsisSystem.DISCARDS)) != 0)
                                Isis.WriteLine("CBWhenReady ignoring this message: p2ps.inSeqn was " + p2ps.inSeqn + "\r\n --" + m);
                            using (new LockAndElevate(IsisSystem.RTS.Lock))
                                IsisSystem.RTS.Discarded++;
                            return;
                        }
                        if (p2ps.outOfOrder.ContainsKey(m.msgid))
                        {
                            if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.P2PLAYER | IsisSystem.DISCARDS)) != 0)
                                Isis.WriteLine("CBWhenReady ignoring this message: p2ps.inSeqn already lists it\r\n --" + m);
                            using (new LockAndElevate(IsisSystem.RTS.Lock))
                                IsisSystem.RTS.Discarded++;
                            return;
                        }

                        if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.P2PLAYER)) != 0)
                            Isis.WriteLine("CBWhenReady invoked (p2p sequencer for sender " + sender + "): msgid " + m.msgid + ", p2ps.inseqn " + p2ps.inSeqn);

                        if (p2ps.outOfOrder.Count() == 0)
                            p2ps.lastCallbackTime = Isis.NOW();
                        p2ps.outOfOrder.Add(m.msgid, new P2PCBList(m, cb));

                        // Deliver messages in order
                        KeyValuePair<int, P2PCBList> kvp;
                        while ((kvp = p2ps.outOfOrder.ElementAtOrDefault(0)).Value != null && kvp.Key - kvp.Value.theMsg.nRaw <= p2ps.inSeqn)
                        {
                            p2ps.outOfOrder.Remove(kvp.Key);
                            if ((kvp.Value.theMsg.nRaw > 0 && kvp.Key - kvp.Value.theMsg.nRaw <= p2ps.inSeqn) || kvp.Key == p2ps.inSeqn)
                            {
                                // If it isn't a duplicate and it is in order with no gaps, deliver it
                                if (kvp.Value.theCB != null)
                                {
                                    if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.P2PLAYER)) != 0)
                                    {
                                        Msg dqm = kvp.Value.theMsg;
                                        Isis.WriteLine("CBWhenReady dequeue and deliver: " + dqm.sender + "::" + dqm.vid + ":" + dqm.msgid + ", p2ps.inseqn " + p2ps.inSeqn);
                                    }
                                    p2ps.callbacksToDo.Add(kvp);
                                }
                                if (kvp.Value.theMsg.nRaw > 0)
                                    p2ps.inSeqn = kvp.Key;
                                else if (p2ps.inSeqn != kvp.Key)
                                    throw new IsisException("P2PS.inseqn for " + p2ps.dest + " jumped from " + p2ps.inSeqn + " to " + kvp.Key);
                                p2ps.inSeqn++;
                            }
                            else if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.P2PLAYER | IsisSystem.DISCARDS)) != 0)
                                Isis.WriteLine("CBWhenReady ignoring this message: when removing it from p2p queue, noticed that p2ps.inseqn=" + p2ps.inSeqn + "but msg = " + m);
                        }
                        if (p2ps.callbacksToDo.Count() == 0 && kvp.Value != null && kvp.Key > p2ps.inSeqn)
                            for (int seqn = p2ps.inSeqn; seqn < Math.Min(p2ps.inSeqn + 2, kvp.Key); seqn++)
                                SendP2PNack(sender, seqn);
                    }
                    if (p2ps.callbacksToDo.Count() == 0)
                        return;

                    List<KeyValuePair<int, P2PCBList>> cblist;
                    while (true)
                    {
                        using (new LockAndElevate(p2ps.Lock))
                        {
                            cblist = p2ps.callbacksToDo;
                            if (cblist.Count() == 0)
                                break;
                            p2ps.callbacksToDo = new List<KeyValuePair<int, P2PCBList>>();
                        }
                        foreach (KeyValuePair<int, P2PCBList> kvp in cblist)
                        {
                            if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.P2PLAYER)) != 0)
                                Isis.WriteLine("CBWhenReady calling " + kvp.Value.theCB.Method.Name + " (p2p sequencer for sender " + sender + "): msgid " + kvp.Value.theMsg.vid + ":" + kvp.Value.theMsg.msgid + ", p2ps.inseqn " + p2ps.inSeqn);
                            using (new LockAndElevate(p2ps.Lock))
                                p2ps.lastCallbackTime = Isis.NOW();
                            kvp.Value.theCB(kvp.Value.theMsg);
                        }
                        using (new LockAndElevate(p2ps.Lock))
                        {
                            if (p2ps.outOfOrder.Count() == 0)
                                p2ps.lastCallbackTime = 0;
                        }
                    }
                }
            }

            internal static void remoteFailed(Address who)
            {
                if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.P2PLAYER)) != 0)
                    Isis.WriteLine("remoteFailed called for " + who);
                using (new LockAndElevate(PSListLock))
                    PSIndex.Remove(who);
            }
        }

        // This is for internal use by Msg.Send(), which is a public interface
        internal static void SendP2P(byte type, Address destProc, Group g, byte[] buffer, bool localSender)
        {
            SendP2P(type, destProc, g, buffer, localSender, null);
        }

        internal static void SendP2P(byte type, Address destProc, Group g, byte[] buffer, bool localSender, Action dcb)
        {
            if ((IsisSystem.Debug & IsisSystem.P2PLAYER) != 0)
                Isis.WriteLine("Calling SendP2P from the internal SendP2P method");
            SendP2P(type, destProc, g, 0, P2PSequencer.NextP2PSeqn("sendp2p", destProc), buffer, localSender, dcb, null);
        }

        internal static byte[] QueryP2P(byte type, Address destProc, Timeout timeout, Group g, byte[] buffer)
        {
            return QueryP2P(type, destProc, timeout, g, 0, P2PSequencer.NextP2PSeqn("queryp2p", destProc), Msg.NEEDSREPLY, buffer);
        }


        internal static byte[] QueryP2P(byte type, Address destProc, Group g, byte[] buffer)
        {
            return QueryP2P(type, destProc, new Timeout(Isis.ISIS_DEFAULTTIMEOUT, Timeout.TO_ABORTREPLY, "QueryP2P/1"), g, 0, P2PSequencer.NextP2PSeqn("queryp2p/2", destProc), Msg.NEEDSREPLY, buffer);
        }

        internal static void SendP2P(byte type, Address destProc, Group g, int vid, int MsgID, byte[] buffer, bool localSender, Action dcb, Msg replyTo)
        {
            Address gaddr = (g == null) ? Isis.NULLADDRESS : g.gaddr;
            if ((IsisSystem.Debug & IsisSystem.P2PLAYER) != 0)
                Isis.WriteLine("SendP2P: type " + type + " dest " + destProc + " group " + (g == null ? "<null>" : g.gname) + " id " + vid + ":" + MsgID);
            doSend(true, my_p2psocket, g, type, destProc, Msg.NewMsgAsBArray(Isis.my_address, gaddr, vid, MsgID, 0L, 0, 0, 0, buffer), vid, MsgID, localSender, dcb);
        }

        internal static byte[] QueryP2P(byte type, Address destProc, Group g, int vid, int MsgID, byte flags, byte[] buffer)
        {
            return QueryP2P(type, destProc, new Timeout(Isis.ISIS_DEFAULTTIMEOUT, Timeout.TO_ABORTREPLY, "QUERYP2P/2"), g, vid, MsgID, flags, buffer);
        }

        internal static byte[] QueryP2P(byte type, Address destProc, Timeout timeout, Group g, int vid, int MsgID, byte flags, byte[] buffer)
        {
            Address gaddr = (g == null || type == Msg.ISPUREP2P) ? Isis.NULLADDRESS : g.gaddr;
            bool ExpectCiphered = (g == null) ? false : ((g.flags & Group.G_SECURE) != 0);
            if (ExpectCiphered)
                flags |= Msg.ENCIPHEREDREPLY;
            if ((IsisSystem.Debug & (IsisSystem.REPLYWAIT | IsisSystem.P2PLAYER)) != 0)
                Isis.WriteLine("QueryP2P: Registering a wait structure... " + vid + ":" + MsgID);
            AwaitReplies.ReplyInfo ri = AwaitReplies.registerP2PWait(g, vid, MsgID, ExpectCiphered, destProc, timeout.when);
            if ((IsisSystem.Debug & (IsisSystem.REPLYWAIT | IsisSystem.P2PLAYER)) != 0)
                Isis.WriteLine("QueryP2P: Sending the query...");
            doSend(true, my_p2psocket, g, type, destProc, Msg.NewMsgAsBArray(Isis.my_address, gaddr, vid, MsgID, 0L, 0, 0, flags, buffer), vid, MsgID, true, null);
            if ((IsisSystem.Debug & (IsisSystem.REPLYWAIT | IsisSystem.P2PLAYER)) != 0)
                Isis.WriteLine("QueryP2P: Collecting the replies...");
            AwaitReplies.awaitReplies(ri, g, timeout, null);
            if ((IsisSystem.Debug & IsisSystem.REPLYWAIT) != 0)
                Isis.WriteLine("QueryP2P: Collected replies, list contains " + ri.rdvReplies.Count());
            if ((g.flags & Group.G_SECURE) != 0)
                g.DecipherReplies(ri);
            if (ri.rdvReplies.Count() > 0)
                return ri.rdvReplies[0];
            return new byte[0];
        }

        internal static void SendGroup(byte type, Group g, int vid, int MsgID, byte flag, byte[] buffer, int nRaw)
        {
            long Lid = 0;
            if (g != null)
                Lid = g.lookupLoggingId();
            doSend(false, g.my_socket, g, type, Isis.NULLADDRESS, Msg.NewMsgAsBArray(Isis.my_address, g.gaddr, vid, MsgID, Lid, nRaw, 0, flag, buffer), vid, MsgID, true, null);
        }

        internal static void SendGroup(byte type, Group g, Msg m, bool localSender)
        {
            doSend(false, g.my_socket, g, type, Isis.NULLADDRESS, Msg.NewMsgAsBArray(m.sender, g.gaddr, m.vid, m.msgid, m.Lid, 0, 0, m.flags, m.payload), m.vid, m.msgid, localSender, null);
        }

        internal static void SendP2P(byte type, Address destProc, Group g, Msg m, bool localSender)
        {
            doSend(true, g.my_socket, g, type, destProc, Msg.NewMsgAsBArray(m.sender, g.gaddr, m.vid, m.msgid, m.Lid, 0, 0, m.flags, m.payload), m.vid, m.msgid, localSender, null);
        }

        internal static void Forward(Address who, Address gaddr, Msg m)
        {
            if (who == null || gaddr == null || m == null)
                throw new IsisException("Bum arg to Forward");
            doSend(true, my_p2psocket, null, Msg.FIFOCAST, who, m.toBArray(), m.vid, m.msgid, m.sender.isMyAddress(), null);
        }

        internal const byte ISCOMP = 0x01;
        internal const byte ISFRAG = 0x02;
        internal const byte SENTBYFRAGGER = 0x04;

        internal static void doSend(bool p2p, object s, Group g, byte type, Address dest, byte[] buffer, int vid, int MsgID, bool localSender, Action dcb)
        {
            if (!IsisSystem.IsisActive && IsisSystem.IsisWasActive)
                return;
            if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                using (new LockAndElevate(ackInfoLock))
                    ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: doSend p2p=" + p2p + ", g=" + (g == null ? "null" : g.gname) + ", " + dest + "::" + vid + ":" + MsgID + "\r\n");

            if (s == null && !Isis.ISIS_TCP_ONLY)
                // This can happen during OOB transfer of the initial view when nodes are joining
                s = ReliableSender.my_p2psocket;

            if (g != null)
            {
                Group.tokenInfo theToken;
                using (new LockAndElevate(g.TokenLock))
                    theToken = g.theToken;
                if ((type == Msg.FIFOCAST || type == Msg.RAWFIFOCAST || type == Msg.UNORDERED) && (g.flags & Group.G_ISLARGE) != 0 && theToken != null && !theToken.IAmLgOwner)
                    throw new IsisException("doSend was asked to multicast to a large group, but isn't the owner!");
            }

            byte[] original_buffer = null;
            Address gaddr = (g == null || type == Msg.ISPUREP2P) ? Isis.NULLADDRESS : g.gaddr;
            // First handle loop-back sends
            if (p2p && dest.isMyAddress())
            {
                if (type != Msg.ISPING && (IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0)
                    Isis.WriteLine("[0] Loopback and return in doSend");
                LoopBack(type, gaddr, Isis.my_address, 0, new Msg(buffer));
                if (dcb != null)
                    dcb();
                if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                    using (new LockAndElevate(ackInfoLock))
                        ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: doSend loopback case\r\n");
                return;
            }
            if (!p2p && g != null && g.theView != null && g.theView.GetMyRank() != -1)
                original_buffer = buffer;

            // Now send to remote destinations, if there were some
            byte mode = 0;
            byte[] bufferAsGiven = null;
            if (buffer.Length > Isis.ISIS_MAXMSGLEN)
            {
                byte[] bc;
                if (Isis.ISIS_DONT_COMPRESS == false && (bc = Compress(buffer)) != null)
                {
                    bufferAsGiven = buffer;
                    buffer = bc;
                    mode |= ISCOMP;
                }
            }
            socketSend(p2p, s, dest, g, type, mode, buffer, bufferAsGiven, vid, MsgID, localSender, dcb);
            if (original_buffer != null)
            {
                if (type != Msg.ISPING && (IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0)
                    Isis.WriteLine("[2] Loopback and return in doSend");
                LoopBack(type, gaddr, Isis.my_address, 0, new Msg(original_buffer));
                if (dcb != null)
                    dcb();
            }

            if (type != Msg.ISPING && (IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0)
                Isis.WriteLine("finished in doSend");
        }

        internal static bool doSendStability(Group g)
        {
            if ((g.flags & Group.G_ISLARGE) != 0 && (!g.HasFirstView || g.IAmLeader()))
                return true;
            using (new LockAndElevate(g.GroupFlagsLock))
                if ((g.flags & Group.G_SENDINGSTABILITY) != 0 || (Isis.NOW() - g.SentStableAt) < 100)
                    return false;
                else
                    g.flags |= Group.G_SENDINGSTABILITY;
            // Do the action as a thread because we're called from the resender thread
            Thread st = new Thread(delegate()
            {
                try
                {
                    MCMDSocket s = g.my_socket;
                    if ((g.GroupOpen || !g.WasOpen) && s != null && g.HasFirstView)
                    {
                        View v;
                        using (new LockAndElevate(g.ViewLock))
                            v = g.theView;
                        byte[] mba = Msg.NewMsgAsBArray(Isis.my_address, g.gaddr, 0, -1, 0L, 0, 0, (byte)0, new byte[0]);
                        if (!Isis.ISIS_UNICAST_ONLY)
                        {
                            using (new LockAndElevate(IsisSystem.RTS.Lock))
                                IsisSystem.RTS.StabilitySent++;
                            socketSend(false, s, g.gaddr, g, Msg.STABILITYINFO, 0, mba, null, 0, -1, true, null);
                        }
                        else foreach (Address dest in v.members)
                                if (!dest.isMyAddress())
                                    socketSend(true, my_p2psocket, dest, g, Msg.STABILITYINFO, 0, mba, null, 0, -1, true, null);
                    }
                    using (new LockAndElevate(g.GroupFlagsLock))
                    {
                        g.flags &= ~Group.G_SENDINGSTABILITY;
                        g.SentStableAt = Isis.NOW();
                        g.MaxBacklogSent = 0;
                    }
                }
                catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                IsisSystem.ThreadTerminationMagic();
            });
            st.Name = "<" + g.gname + ">: send stability";
            st.Priority = ThreadPriority.Highest;
            st.Start();
            return true;
        }

        internal static int Counter;

        internal static void socketSend(bool p2p, object os, Address dest, Group g, byte type, byte code, byte[] originalBuffer, byte[] bufferAsGiven, int vid, int MsgID, bool localSender, Action dcb)
        {
            bool isGroupAddress = (type == Msg.FIFOCAST || type == Msg.RAWFIFOCAST || type == Msg.UNORDERED) && (Group.doLookup(dest) != null || Group.TrackingProxyLookup(dest) != null);
            if (os == null)
                throw new IsisException("null socket in Isis socketSend");
            Type sType = os.GetType();
            Address gaddr;
            int minStable;
            if (g == null || type == Msg.ISPUREP2P)
            {
                gaddr = Isis.NULLADDRESS;
                minStable = -1;
            }
            else
            {
                gaddr = g.gaddr;
                minStable = g.getMinStable();
            }

            int myCounter;

            using (new LockAndElevate(PendingSendBufferLock))
                myCounter = ++Counter;

            if (g != null && g.SIFList != null)
            {
                g.SIFList.Add(new object[16] { p2p, os, dest, g, type, code, originalBuffer, vid, MsgID, localSender, dcb, isGroupAddress, sType, gaddr, minStable, myCounter });
                return;
            }
            if (originalBuffer.Length > Isis.ISIS_MUSTFRAGMENT && (code & SENTBYFRAGGER) == 0 && !Isis.ISIS_TCP_ONLY)
            {
                if (!Isis.BigTimeouts && originalBuffer.Length > 500000)
                {
                    Isis.BigTimeouts = true;
                    Isis.ISIS_DEFAULTTIMEOUT = Isis.ISIS_DEFAULTTIMEOUT * 2;
                }
                if (p2p && !localSender)
                    throw new IsisException("Calling SendInFrags for a non-local p2p message from " + Isis.ExtractStackTrace());
                if (g != null)
                {
                    if (g.SIFList != null)
                        throw new IsisException("SIFList should be null");
                    g.SIFList = new List<object[]>();
                }
                originalBuffer = Group.SendInFrags(p2p, dest, g, originalBuffer, bufferAsGiven);
                code |= ISFRAG;
                if (g != null)
                {
                    g.SIFList.Add(new object[16] { p2p, os, dest, g, type, code, originalBuffer, vid, MsgID, localSender, dcb, isGroupAddress, sType, gaddr, minStable, myCounter });
                    List<Object[]> toSend = g.SIFList;
                    g.SIFList = null;
                    // Send the SIFList fragments without SIFLock
                    new Thread(delegate()
                    {
                        try
                        {
                            Thread.CurrentThread.Name = "<" + g.gname + "> SIFList sender";
                            foreach (object[] obs in toSend)
                                //doSendBody(p2p, os, dest, g, type, code, originalBuffer, vid, MsgID, localSender, dcb, isGroupAddress, sType, gaddr, minStable, myCounter)
                                socketSendAction((bool)obs[0], obs[1], (Address)obs[2], (Group)obs[3], (byte)obs[4], (byte)obs[5], (byte[])obs[6],
                                    (int)obs[7], (int)obs[8], (bool)obs[9], (Action)obs[10], (bool)obs[11], (Type)obs[12], (Address)obs[13], (int)obs[14], (int)obs[15]);
                        }
                        catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                        IsisSystem.ThreadTerminationMagic();
                    }).Start();
                    return;
                }
            }
            socketSendAction(p2p, os, dest, g, type, code, originalBuffer, vid, MsgID, localSender, dcb, isGroupAddress, sType, gaddr, minStable, myCounter);
        }

        private static void socketSendAction(bool p2p, object os, Address dest, Group g, byte type, byte code, byte[] originalBuffer, int vid, int MsgID, bool localSender, Action dcb, bool isGroupAddress, Type sType, Address gaddr, int minStable, int myCounter)
        {
            if (!dest.isGroupAddress() && !dest.isNull())
                p2p = true;
            byte[] buffer = Msg.toBArray(type, code, myCounter, Isis.my_address, dest, gaddr, minStable, FlowControl.Backlog(g), originalBuffer);
            if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                using (new LockAndElevate(ackInfoLock))
                    ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: socketSend " + ((vid != 0 || MsgID != -1) ? " calling " : " skipping ") + " remember for " + dest + "::" + vid + ":" + MsgID + "\r\n");
            if ((vid != 0 || MsgID != -1) && type != Msg.ISRAWGRPP2P && type != Msg.ISRAWREPLY && type != Msg.RAWFIFOCAST)
                Remember(p2p, dest, g, type, code, buffer, vid, MsgID, localSender, isGroupAddress, myCounter, dcb);

            if (g != Isis.ISISMEMBERS && g != Isis.ORACLE && Isis.ISIS_TCP_ONLY)
            {
                TunnelMsg(p2p, dest, buffer, gaddr);
                return;
            }

            if (type != Msg.ISPING && (IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0)
                Isis.WriteLine("Send preparing to " + (p2p ? "send pt-to-pt" : "multicast") + " outgoming msg: type " + type + ", seqn" + myCounter +
                        ", sender " + Isis.my_address + ", dest " + dest + ", gaddr " + gaddr + ", minStable " + minStable + ", buffer len " + buffer.Length);
            if (dest != null && (dest.isMyAddress() || (!p2p && Group.doLookup(dest) != null)))
            {
                // Loop a copy back to myself because the ReliableSender ignores messages from me to me.  Lacking this,
                // if the SendTo fails at the sender side, the message won't loop back and the system will hang
                AddToReceiveBB(buffer, false);
            }
            if ((type != Msg.FIFOCAST && type != Msg.RAWFIFOCAST && type != Msg.UNORDERED) || p2p)
            {
                if (p2p && isGroupAddress)
                    throw new IsisException("Using UNICAST logic to send a multicast to group " + dest + " but should have been a member of that group! (p2p=true, type=" + type + ", group by name=<" + g.gname + ">");
                if (dest.isMyAddress() == false)
                {
                    if (Isis.ISIS_TCP_ONLY)
                    {
                        TunnelMsg(p2p, dest, buffer, gaddr);
                        return;
                    }
                    IPEndPoint remoteEP = new IPEndPoint(dest.home, dest.p2pPort);
                    try
                    {
                        if (os.GetType().Equals(typeof(Socket)))
                        {
                            ReliableSender.CheckLen(buffer);
                            int bs = ((Socket)os).SendTo(buffer, remoteEP);
                            if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                                using (new LockAndElevate(ackInfoLock))
                                    ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: Primary Send to " + remoteEP + " for " + dest + "\r\n");
                            if ((Socket)os != my_acksocket)
                                using (new LockAndElevate(IsisSystem.RTS.Lock))
                                {
                                    IsisSystem.RTS.UDPBsent += bs;
                                    IsisSystem.RTS.UDPsent++;
                                }
                        }
                        else if (Isis.ISIS_UNICAST_ONLY == false && ((MCMDSocket)os).UseUnicast() == false)
                            ((MCMDSocket)os).SendTo(buffer);
                        else
                        {
                            if (g.HasFirstView == false)
                                return;
                            Multicast(g, buffer);
                        }
                    }
                    catch { }
                }
                return;
            }
            if ((IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0)
                Isis.WriteLine("SocketSend CAST: " + buffer.Length + " bytes");
            int lcount = 0;
        again:
            if (Isis.ISIS_UNICAST_ONLY == false && Isis.ISIS_TCP_ONLY == false
                && (sType.Equals(typeof(MCMDSocket)) == false || ((MCMDSocket)os).UseUnicast() == false) && g != null && g.myPhysIPAddr != MCMDSocket.USEUNICAST)
            {
                if (sType.Equals(typeof(Socket)))
                {
                    ReliableSender.CheckLen(buffer);
                    int bs = ((Socket)os).SendTo(buffer, g.my_socket.GetRemoteEP());
                    if ((Socket)os != my_acksocket)
                        using (new LockAndElevate(IsisSystem.RTS.Lock))
                        {
                            IsisSystem.RTS.UDPBsent += bs;
                            IsisSystem.RTS.UDPsent++;
                        }
                }
                else
                    try
                    {
                        // Could throw an MCMD exception if we switch to Unicast mode just at this instant
                        ((MCMDSocket)os).SendTo(buffer);
                    }
                    catch (MCMDException)
                    {
                        if (lcount++ == 0)
                            goto again;
                        throw new IsisException("Stuck looping on UseUnicast");
                    }

            }
            else if (g.gname.Equals("ORACLE") && g.HasFirstView == false && Isis.ISIS_HOSTS_IPADDRS != null)
            {
                if (Isis.ISIS_HOSTS_IPADDRS == null)
                    throw new IsisException("Unable to contact ORACLE: In USEUNICAST/TCP_ONLY mode, ISIS_HOSTS must list locations where ORACLE servers are running");
                // Boostrap via p2p rendezvous with the hosts in ISIS_HOSTS
                foreach (IPAddress ipa in Isis.ISIS_HOSTS_IPADDRS)
                {
                    IPEndPoint remoteEP = new IPEndPoint(ipa, (Isis.ISIS_TCP_ONLY || Isis.ISIS_UNICAST_ONLY) ? Isis.ISIS_DEFAULT_PORTNOp : Isis.ISIS_GROUPPORT);
                    if (Isis.ISIS_TCP_ONLY)
                        TCPSendTo(Isis.NULLADDRESS, buffer, remoteEP, SEEKU);
                    else
                    {
                        ReliableSender.CheckLen(buffer);
                        int bs = my_p2psocket.SendTo(buffer, remoteEP);
                        using (new LockAndElevate(IsisSystem.RTS.Lock))
                        {
                            IsisSystem.RTS.UDPBsent += bs;
                            IsisSystem.RTS.UDPsent++;
                        }
                    }
                }
                return;
            }
            else
            {
                if (g.HasFirstView == false)
                    return;
                Multicast(g, buffer);
            }
        }

        internal static void AddToReceiveBB(byte[] buffer, bool nonBlocking)
        {
            RecvBB.put(buffer, nonBlocking);
        }

        internal static void Multicast(Group g, byte[] buffer)
        {
            View theView;
            using (new LockAndElevate(g.ViewLock))
                theView = g.theView;
            if (theView.members.Length > Isis.ISIS_MAXDIRECTSENDS)
            {
                Isis.ISISMEMBERS.IPMCTunnel(g.gaddr, buffer);
                return;
            }
            try
            {
                foreach (Address dest in theView.members)
                {
                    if (dest.isMyAddress())
                        continue;
                    IPEndPoint remoteEP = new IPEndPoint(dest.home, Isis.ISIS_TCP_ONLY ? Isis.ISIS_DEFAULT_PORTNOp : dest.p2pPort);
                    if (Isis.ISIS_TCP_ONLY)
                        TCPSendTo(dest, buffer, remoteEP, RECVBB);
                    else
                    {
                        ReliableSender.CheckLen(buffer);
                        int bs = my_p2psocket.SendTo(buffer, remoteEP);
                        using (new LockAndElevate(IsisSystem.RTS.Lock))
                        {
                            IsisSystem.RTS.UDPBsent += bs;
                            IsisSystem.RTS.UDPsent++;
                        }
                    }
                }
            }
            catch (SocketException) { return; }
            catch (ObjectDisposedException) { return; }
        }

        private static void Remember(bool p2p, Address dest, Group g, byte type, byte code, byte[] buffer, int vid, int MsgID, bool localSender, bool isGroupAddress, int myCounter, Action dcb)
        {
            activeMsg am = null;
            if (dest.isMyAddress() || (p2p && !localSender))
                return;
            if (localSender && g != null && (g.flags & Group.G_ISLARGE) != 0)
                am = new activeMsg(g, vid, MsgID);
            int cnt = 0;
            using (new LockAndElevate(Group.GroupRIPLock))
                if (!p2p && (g != null && Group.GroupRIPList.Contains(g.gaddr)) || (isGroupAddress && Group.GroupRIPList.Contains(dest)))
                    return;
            while (g != null && !p2p && !g.HasFirstView)
            {
                Thread.Sleep(1000);
                if (++cnt == 60)
                    throw new IsisException("Remember: group <" + g.gname + "> lacks first view");
            }
            if (g != null && !g.GroupOpen && (!p2p || !localSender))
                return;
            int rtdelay = 250;
            if ((code & ISFRAG) != 0)
                rtdelay += (int)(5 * (buffer.Length / Isis.ISIS_FRAGLEN));
            if (p2p)
            {
                using (new LockAndElevate(PendingSendBufferLock))
                {
                    if (dest != null && !isGroupAddress)
                    {
                        using (new LockAndElevate(Isis.RIPLock))
                            if (!Isis.RIPList.Contains(dest))
                            {
                                if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                                    using (new LockAndElevate(ackInfoLock))
                                        ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: Remember appending to P2P PendingSendBuffer " + dest + "::" + vid + ":" + MsgID + "\r\n");

                                LinkedListNode<MsgDesc> mdnode = PendingSendBuffer.Last;
                                while (mdnode != null && mdnode.Value.UID > myCounter)
                                    mdnode = mdnode.Previous;
                                MsgDesc md = new MsgDesc(type, Isis.ISIS_MAXRETRIES, null, dest, true, false, myCounter, buffer, vid, MsgID, rtdelay, 1, 1, localSender, dcb, am);
                                if (am != null)
                                    am.SentCnt++;
                                if (mdnode != null)
                                    PendingSendBuffer.AddAfter(mdnode, md);
                                else
                                    PendingSendBuffer.AddFirst(md);
                                dcb = null;
                            }

                    }
                }
                if (dcb != null)
                    dcb();
                if (am != null)
                    using (new LockAndElevate(activeMsgLock))
                        activeMsgs.Add(am);
                return;
            }
            if (g == null)
            {
                if (!p2p) throw new IsisException("Remember: g=null but !p2p");
                if (dcb != null)
                    dcb();
                return;
            }
            View v = g.theView;
            if ((g.flags & Group.G_ISLARGE) != 0)
            {
                using (new LockAndElevate(Group.GroupRIPLock))
                {
                    if (Group.GroupRIPList.Contains(g.gaddr))
                        return;
                }
                if (type == Msg.FIFOCAST || type == Msg.RAWFIFOCAST || type == Msg.UNORDERED)
                {
                    if ((IsisSystem.Debug & (IsisSystem.TOKENLOGIC | IsisSystem.FLUSHING)) != 0)
                        Isis.WriteLine("Add to LgPending Send Buffer: msgid=" + MsgID);
                    using (new LockAndElevate(PendingSendBufferLock))
                        if (g.GroupOpen || !g.WasOpen)
                            LgPendingSendBuffer.AddLast(new MsgDesc(type, 5, g, g.gaddr, false, true, MsgID, buffer, vid, MsgID, rtdelay * 10, 1, 5, localSender, dcb, am));
                    if ((IsisSystem.Debug & IsisSystem.TOKENSTABILITY) != 0)
                        Isis.WriteLine("large-group remember ADDED to lgPendingSendQueue: " + (p2p ? "p2p" : "multicast") + " is of type=" + type + ", group=<" + g.gname + ">, MsgID=" + MsgID + ", rtdelay=" + (rtdelay * 3));
                    return;
                }
                else if (dcb != null)
                    dcb();
                if ((IsisSystem.Debug & IsisSystem.TOKENSTABILITY) != 0)
                    Isis.WriteLine("large-group remember DIDN'T ADD to lgPendingSendQueue: " + (p2p ? "p2p" : "multicast") + " is of type=" + type + ", group=<" + g.gname + ">, MsgID=" + MsgID + ", rtdelay=" + (rtdelay * 3));
                return;
            }
            if ((IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0)
                Isis.WriteLine("PendingSendBuffer.Add: sender=" + Isis.my_address + ", msgid=" + vid + ":" + MsgID + ", ACKID " + myCounter + ", theView.vid=" +
                    v.viewid + ", dests=" + Address.VectorToString(v.members));
            Address[] dests = v.members;
            using (new LockAndElevate(PendingSendBufferLock))
            {
                LinkedListNode<MsgDesc> mdnode = PendingSendBuffer.Last;
                while (mdnode != null && mdnode.Value.UID > myCounter)
                    mdnode = mdnode.Previous;
                using (new LockAndElevate(Isis.RIPLock))
                    for (int i = 0; i < dests.Length; i++)
                    {
                        Address d = dests[i];
                        if (d.isMyAddress() || v.hasFailed[i])
                            continue;
                        bool seemsHealthy = false;
                        if (!Isis.RIPList.Contains(dest))
                            seemsHealthy = true;
                        if (seemsHealthy)
                        {
                            if (!Isis.RIPList.Contains(d))
                            {
                                MsgDesc md = new MsgDesc(type, Isis.ISIS_MAXRETRIES, g, d, false, false, myCounter, buffer, vid, MsgID, 250, 1, dests.Length, localSender, dcb, am);
                                if (am != null)
                                    am.SentCnt++;
                                if (mdnode != null)
                                    PendingSendBuffer.AddAfter(mdnode, md);
                                else
                                    PendingSendBuffer.AddFirst(md);
                            }
                            else if (dcb != null)
                                dcb();
                        }
                    }
            }
            if (am != null)
                using (new LockAndElevate(activeMsgLock))
                    activeMsgs.Add(am);
        }

        private static void TunnelMsg(bool p2p, Address dest, byte[] buffer, Address gaddr)
        {
            if ((IsisSystem.Debug & IsisSystem.TCPOVERLAY) != 0)
                Isis.WriteLine("TunnelMsg: p2p=" + p2p + ", dest=" + dest + ", gaddr=" + gaddr + ", buffer length = " + buffer.Length);
            if (p2p)
            {
                if (dest.isMyAddress())
                {
                    ReliableSender.AddToReceiveBB(buffer, true);
                    return;
                }
                if (ReliableSender.TCPSendIfKnown(dest, buffer, ReliableSender.RECVBB))
                    return;
            }
            if (Isis.ISISMEMBERS == null)
                // Only happens for the very first few seconds during startup, just drop the message
                return;
            if (p2p)
                Isis.ISISMEMBERS.UDPTunnel(dest, buffer, 0);
            else
                Isis.ISISMEMBERS.IPMCTunnel(gaddr, buffer);
            return;
        }

        delegate void RDel(byte type, byte code, int seqn, Address sender, Address dest, Address gaddr, int minStable, int myBacklog, byte[] buf);

        internal static bool recieverRunning = false;

        internal static void SReceive(object os)
        {
            while (!IsisSystem.IsisActive)
                Thread.Sleep(250);
            if (!Isis.ISIS_TCP_ONLY && !recieverRunning)
            {
                recieverRunning = true;
                RecvBB = new BoundedBuffer("BB:P2P", 1024, ILock.LLP2PB, 0, 1);
                Thread t = new Thread(delegate()
                {
                    while (IsisSystem.IsisActive)
                    {
                        IsisSystem.RTS.ThreadCntrs[21]++;
                        try
                        {
                            byte[] buffer = RecieveOne(os);
                            AddToReceiveBB(buffer, true);
                        }
                        catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                        catch (Exception e)
                        {
                            if (IsisSystem.IsisActive)
                                throw new IsisException("Dedicated receiver thread threw Exception: " + e);
                        }
                    }
                    IsisSystem.ThreadTerminationMagic();
                });
                t.Name = "Dedicated P2P receiver thread";
                t.Priority = ThreadPriority.Highest;
                t.Start();
            }
            doReceive(os, null);
        }

        // Sorry about the slightly tortured control flow here.  Added this dedicated high-speed receive thread for the p2p socket
        // and the AckSocket when we saw a few cases of severe packet loss associated with minor scheduling delays.  
        private static byte[] RecieveOne(object os)
        {
            byte[] buffer = new byte[Isis.ISIS_MAXMSGLEN + 1024];
            int len;
            do
            {
                try
                {
                    len = ((Socket)os).Receive(buffer);
                }
                catch (SocketException) { len = 0; }
                catch (IOException) { len = 0; }
                IsisSystem.RTS.ThreadCntrs[30]++;
            }
            while (!Msg.VerifySignature(buffer, 0, len));
            IsisSystem.RTS.ThreadCntrs[31]++;

            using (new LockAndElevate(IsisSystem.RTS.Lock))
            {
                IsisSystem.RTS.UDPrcvd++;
                IsisSystem.RTS.UDPBrcvd += len;
            }
            IsisSystem.RTS.check();
            if (len > Isis.ISIS_MAXMSGLEN)
                Isis.WriteLine("WARNING(a): MAXMSGLEN set to " + Isis.ISIS_MAXMSGLEN + ", but received length=" + len);
            if (len != Isis.ISIS_MAXMSGLEN)
                Isis.ArrayResize<byte>(ref buffer, len);
            if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                using (new LockAndElevate(ackInfoLock))
                {
                    IPEndPoint remoteEP = new IPEndPoint(Isis.my_address.home, Isis.my_address.p2pPort);
                    ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: Receive(" + (((Socket)os == my_p2psocket) ? "p2p-socket" : "group-socket") +
                        "): my IP " + remoteEP + " my addr " + Isis.my_address + "\r\n");
                }
            return buffer;
        }

        internal static void Receive(Group g)
        {
            doReceive(g.my_socket, g);
        }

        internal static BoundedBuffer RecvBB;

        internal static void TCPTunnelSetup()
        {
            RecvBB = new BoundedBuffer("TCP:P2P", 1024, ILock.LLP2PB, 0, 1);
            AckBB = new BoundedBuffer("TCP:Ack", 128, ILock.LLACKB, 0, 1);
        }

        internal class MReplayMe
        {
            internal byte type;
            internal Address sender;
            internal Address gaddr;
            internal int minStable;
            internal Msg msg;

            internal MReplayMe(byte t, Address s, Address g, int ms, Msg m) { type = t; sender = s; gaddr = g; minStable = ms; msg = m; }
        }

        internal static void doReceive(object os, Group g)
        {
            byte[] buffer;
            Isis.receiveThread = Thread.CurrentThread;
            if (g == null)
                using (new LockAndElevate(IsisSystem.RTS.Lock))
                    IsisSystem.RTS.rcvProcessingBeganAt = 0;
            if (os.GetType().Equals(typeof(Socket)))
            {
                if (Isis.ISIS_TCP_ONLY || ((Socket)os).Equals(my_p2psocket))
                    buffer = (byte[])RecvBB.get();
                else
                    buffer = RecieveOne((Socket)os);
                if (buffer == null || (!IsisSystem.IsisActive && IsisSystem.IsisWasActive))
                    return;
            }
            else if ((buffer = ((MCMDSocket)os).Receive()) == null)
            {
                if (g == null)
                    using (new LockAndElevate(IsisSystem.RTS.Lock))
                        IsisSystem.RTS.rcvProcessingBeganAt = 0;
                Thread.Sleep(10);
                return;
            }
            if (g == null)
                using (new LockAndElevate(IsisSystem.RTS.Lock))
                    if (IsisSystem.RTS.rcvProcessingBeganAt != 0)
                        throw new IsisException("Receive thread: more than one caller has g=null");
                    else
                        IsisSystem.RTS.rcvProcessingBeganAt = Isis.NOW();
            //    buffer = Msg.toBArray(type, code, myCounter, Isis.my_address, dest, gaddr, minStable, myBacklog, buffer);
            try
            {
                long before = Isis.NOW();
                Msg.InvokeFromBArray(buffer, (RDel)delegate(byte type, byte code, int UID, Address sender, Address dest, Address gaddr, int minStable, int remoteBacklog, byte[] buf)
                {
                    // Isis.WriteLine("RecvBB.Get: type=" + type + ", code=" + code + ", UID=" + UID + ", sender=" + sender + ", dest=" + dest + ", gaddr=" + gaddr + " minStable=" + minStable + ", remoteBacklog=" + remoteBacklog + ", buf.len=" + buf.Length);
                    ReliableSender.noteRemoteBacklog(sender, remoteBacklog);
                    if (sender == null || dest == null || gaddr == null || buf == null || (g != null && !g.GroupOpen && g.WasOpen))
                    {
                        if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                            Isis.WriteLine("Instantly discarding an incoming multicast:" + sender + ", UID " + UID);
                        using (new LockAndElevate(IsisSystem.RTS.Lock))
                        {
                            IsisSystem.RTS.Discarded++;
                            if (g == null)
                                IsisSystem.RTS.rcvProcessingBeganAt = 0;
                        }
                        if (g != null && (g.GroupOpen || !g.WasOpen))
                            SendAck(g, sender, UID);
                        return;
                    }
                    if (type != Msg.ISPING)
                        if ((IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0)
                            Isis.WriteLine("Receive successfully parsed incoming msg (phys len " + buffer.Length + "): type " + type + ", seqn" + UID
                                + ", sender " + sender + ", dest " + dest + ", gaddr " + gaddr + ", buffer len " + buf.Length);
                    bool fnd = false;
                    using (new LockAndElevate(Isis.RIPLock))
                        if (Isis.RIPList != null && Isis.RIPList.Contains(sender))
                            fnd = true;
                    if (fnd)
                    {
                        SendPoison(sender, Isis.my_address + " found you on the recently deceased process list");
                        return;
                    }
                    justHeardFrom(sender);
                    if ((code & (ISCOMP | ISFRAG)) == ISCOMP)
                        buf = DeCompress(buf);
                    Msg m = new Msg(buf);
                    if ((m.flags & Msg.CIPHER) != 0)
                    {
                        if (g != null && (g.flags & Group.G_SECURE) != 0 && (type == Msg.ISGRPP2P || type == Msg.ISRAWGRPP2P || type == Msg.FIFOCAST || type == Msg.UNORDERED))
                            g.decipherMsg(m);
                        else
                        {
                            if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                                Isis.WriteLine("Discarded incoming multicast (it was encrypted but group isn't secure):" + sender + ", UID " + UID);
                            using (new LockAndElevate(IsisSystem.RTS.Lock))
                            {
                                IsisSystem.RTS.Discarded++;
                                if (g == null)
                                    IsisSystem.RTS.rcvProcessingBeganAt = 0;
                            }
                            return;
                        }
                    }
                    if (m.sender.Equals(sender))
                    {
                        // Need to be careful here: sometimes a message shows up via a relay through some other source.  It would use its own (sender,UID) when sending to us.
                        // Idea, though, is that upon "original receipt" we note the UID the sender used, and then we can later forward this with the UID pre-filled in (the sender
                        // itself won't do so because it picks the UID only after creating the out-form byte[] representation).  Then can use it to ack to the sender as a way to
                        // let it know that yes, I do have his message x:y with UID abc.... 
                        m.UID = UID;
                    }
                    if ((code & ISFRAG) != 0)
                        m.flags |= Msg.FRAGGED;
                    List<MReplayMe> mae = (g == null) ? null : g.MsgArrivedEarly;
                    m.offWire = buffer;
                    if (g == null && IsisSystem.RTS.rcvProcessingBeganAt > 0 && (Isis.NOW() - IsisSystem.RTS.rcvProcessingBeganAt) > 2500)
                        Isis.WriteLine("WARNING: In receive thread needed " + (Isis.NOW() - IsisSystem.RTS.rcvProcessingBeganAt) + " ms to extract message!");
                    if (GotIncoming(type, gaddr, sender, minStable, m, g == null))
                    {
                        if (type != Msg.ISRAWGRPP2P && type != Msg.ISRAWREPLY && type != Msg.RAWFIFOCAST)
                        {
                            // Ack unless it was discarded
                            if (g == null)
                                using (new LockAndElevate(IsisSystem.RTS.Lock))
                                    IsisSystem.RTS.rcvProcessingBeganAt = 0;
                            if (sender.isMyAddress())
                                GotAnAck(sender, UID);
                            else
                            {
                                if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                                    using (new LockAndElevate(ackInfoLock))
                                        ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: Send Ack " + sender + "::" + m.vid + ":" + m.msgid + " -- UID " + UID + ", type=" + type + "\r\n");
                                SendAck(g, sender, UID);
                            }
                        }
                    }
                    else
                    {
                        if (g == null)
                            using (new LockAndElevate(IsisSystem.RTS.Lock))
                                IsisSystem.RTS.rcvProcessingBeganAt = 0;
                        if (mae != null)
                            mae.Add(new MReplayMe(type, sender, gaddr, minStable, m));
                    }

                    if (g == null && IsisSystem.RTS.rcvProcessingBeganAt > 0 && (Isis.NOW() - IsisSystem.RTS.rcvProcessingBeganAt) > 1000)
                        Isis.WriteLine("WARNING: In receive thread needed " + (Isis.NOW() - IsisSystem.RTS.rcvProcessingBeganAt) + " ms to extract message!");
                });
                if (g == null)
                    using (new LockAndElevate(IsisSystem.RTS.Lock))
                        IsisSystem.RTS.rcvProcessingBeganAt = 0;
                if ((IsisSystem.Debug & IsisSystem.DELAYS) != 0 && (Isis.NOW() - before) > 500)
                    Isis.WriteLine("WARNING: LONG DELAY while processing an incoming message (" + (Isis.NOW() - before) + "ms)");
            }
            catch (IsisException)
            {
                if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                    Isis.WriteLine("Instantly discarding an incoming multicast: misformatted");
                using (new LockAndElevate(IsisSystem.RTS.Lock))
                {
                    IsisSystem.RTS.Discarded++;
                    IsisSystem.RTS.rcvProcessingBeganAt = 0;
                }
                Thread.Sleep(10);
                return;
            }
        }

        internal class LoopBackInf
        {
            internal byte lbtype;
            internal Address lbgaddr;
            internal Address lbsender;
            internal int lbminStable;
            internal Msg lbMsg;

            internal LoopBackInf(byte type, Address gaddr, Address sender, int minStable, Msg m)
            {
                lbtype = type;
                lbgaddr = gaddr;
                lbsender = sender;
                lbminStable = minStable;
                lbMsg = m;
            }
        }

        internal static Thread lbThread;
        internal static BoundedBuffer lbBB;
        internal static LockObject lbBufferLock = new LockObject("lbBufferLock", ThreadPriority.Highest);

        internal static void LoopBack(byte type, Address gaddr, Address sender, int minStable, Msg m)
        {
            if ((IsisSystem.Debug & IsisSystem.LOOPBACK) != 0)
                Isis.WriteLine("LoopBack: type=" + type + ", gaddr=" + gaddr + ", " + sender + "::" + m.vid + ":" + m.msgid + ", minStable=" + minStable + "\r\n  " + m);
            using (new LockAndElevate(lbBufferLock))
            {
                if (lbThread == null)
                {
                    lbBB = new BoundedBuffer("Loopback", 1024, ILock.LLLB, -1, -1);
                    lbThread = new Thread(delegate()
                    {
                        try
                        {
                            while (IsisSystem.IsisActive || !IsisSystem.IsisWasActive)
                            {
                                IsisSystem.RTS.ThreadCntrs[22]++;
                                LoopBackInf lbi = (LoopBackInf)lbBB.get();
                                if (lbi == null)
                                    break;
                                if ((IsisSystem.Debug & IsisSystem.LOOPBACK) != 0)
                                    Isis.WriteLine("LoopBack calling GotIncoming: type=" + lbi.lbtype + ", sender=" + lbi.lbsender + ", minStable=" + lbi.lbminStable + ":  " + lbi.lbMsg);
                                GotIncoming(lbi.lbtype, lbi.lbgaddr, lbi.lbsender, lbi.lbminStable, lbi.lbMsg, false);
                            }
                        }
                        catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                        IsisSystem.ThreadTerminationMagic();
                    });
                    lbThread.Name = "Loopback thread";
                    lbThread.Priority = ThreadPriority.AboveNormal;
                    lbThread.Start();
                }
            }
            lbBB.put(new LoopBackInf(type, gaddr, sender, minStable, m));
        }

        internal static int LBBcnt()
        {
            return BoundedBuffer.FullSlots(lbBB);
        }

        internal static bool GotIncoming(byte type, Address gaddr, Address sender, int minStable, Msg m, bool trackEvents)
        {
            if ((type == Msg.FIFOCAST || type == Msg.UNORDERED) && gaddr.isNull())
                gaddr = m.dest;
            Group g = Group.doLookup(gaddr);
            if (g == null || (g.flags & Group.G_ISLARGE) == 0)
                m.offWire = null;
            return doGotIncoming(type, gaddr, sender, minStable, m, g, trackEvents);
        }

        internal static List<Address> knownWorkers = new List<Address>();

        internal static bool doGotIncoming(byte type, Address gaddr, Address sender, int minStable, Msg m, Group g, bool trackEvents)
        {
            if (!IsisSystem.IsisActive && IsisSystem.IsisWasActive)
                return false;
            switch (type)
            {
                default:
                    Isis.WriteLine("WARNING: Received a message with unknown type: " + type);
                    break;

                case Msg.ISPUREP2P:
                    P2PSequencer.P2PCBWhenReady(m.sender, m, P2P.DeliverP2P);
                    break;

                case Msg.ISGRPP2P:
                case Msg.ISRAWGRPP2P:
                    if (g != null)
                    {
                        g.isStable(sender, minStable);
                        g.CheckForFutureViews(m);
                        // This special field is used to ensure that a P2P message in a group won't be delivered until the group 
                        // reaches the correct view id number
                        m.theGroup = g;
                        P2PSequencer.P2PCBWhenReady(m.sender, m, g.enqueueForGroupDelivery);
                    }
                    else
                    {
                        P2PSequencer.P2PCBWhenReady(m.sender, m, null);
                        bool stashIt = false;
                        using (new LockAndElevate(Group.GroupRIPLock))
                            if (!Group.GroupRIPList.Contains(gaddr))
                                stashIt = true;
                        if (stashIt)
                        {
                            // Tuck away for replay after the group gets created
                            Group.stashMsg(sender, gaddr, minStable, m);
                            return false;
                        }
                    }
                    break;

                case Msg.ISPING:
                case Msg.ISFILLER:
                    P2PSequencer.P2PCBWhenReady(m.sender, m, null);
                    break;

                case Msg.ISREPLY:
                case Msg.ISRAWREPLY:
                    P2PSequencer.P2PCBWhenReady(m.sender, m, AwaitReplies.gotReply);
                    break;

                case Msg.ISTOKEN:
                    P2PSequencer.P2PCBWhenReady(m.sender, m, ReliableSender.gotToken);
                    break;

                case Msg.ISLOOPBACK:
                case Msg.REDELIVERY:
                    g = Group.doLookup(gaddr);
                    if (g != null)
                    {
                        g.CheckForFutureViews(m);
                        if ((IsisSystem.Debug & IsisSystem.DELIVERY) != 0)
                            Isis.WriteLine("<" + g.gname + ">: loopback delivering " + m);
                        g.doDeliveryCallbacks(m, "Looped Back from SortThenDeliverInOrder", type);
                    }
                    else
                    {
                        if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                            Isis.WriteLine("Discarding a looped-back message: this member no longer belongs to the group");
                        using (new LockAndElevate(IsisSystem.RTS.Lock))
                            IsisSystem.RTS.Discarded++;
                    }
                    break;

                case Msg.STABILITYINFO:
                    using (new LockAndElevate(IsisSystem.RTS.Lock))
                        IsisSystem.RTS.StabilityRcvd++;
                    if (g != null && ((g.flags & Group.G_ISLARGE) == 0 || g.HasFirstView))
                        g.isStable(sender, minStable);
                    else
                    {
                        if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.DISCARDS)) != 0)
                            Isis.WriteLine("Discarding an incoming STABILITYINFO message: I'm not a member of group " + gaddr);
                        using (new LockAndElevate(IsisSystem.RTS.Lock))
                            IsisSystem.RTS.Discarded++;
                        return false;
                    }
                    break;

                case Msg.FIFOCAST:
                case Msg.UNORDERED:
                    if (!IsisSystem.IsisRestarting)
                        using (new LockAndElevate(Group.GroupRIPLock))
                            if (Group.GroupRIPList.Contains(gaddr))
                            {
                                if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.DISCARDS)) != 0)
                                    Isis.WriteLine("doGotIncoming discarding " + m.sender + "::" + m.vid + ":" + m.msgid + ", because GroupRIPList=[" + Address.VectorToString(Group.GroupRIPList.ToArray()) + "] contains " + gaddr);
                                return true;
                            }
                    if (g != null && ((g.flags & Group.G_ISLARGE) == 0 || g.HasFirstView))
                    {
                        // Process it, unless I'm not yet ready to do so.  The regular group logic can handle "early" messages, but the
                        // large-group code lacks an equivalent mechanism; it can't handle messages until it knows the first view and builds
                        // the first local token (g.theView.theToken).  But large groups don't ack messages in the same way as
                        // regular ones.  Thus for a regular group, we can pass in an early message; for a large group, we can discard
                        // it safely at this stage, knowing that the token-driven retransmission logic will kick in and resend the
                        // missing one, thus filling the gap (if needed -- after all, early messages might not even be for the view
                        // to which I'll end up belonging
                        ++g.rcvdMcastsCnt;
                        g.isStable(sender, minStable);
                        if ((g.flags & Group.G_SECURE) != 0)
                            g.decipherMsg(m);
                        g.CheckForFutureViews(m);
                        m.type = type;
                        return g.GotAMsg(m, type, "doGotIncoming");
                    }
                    else
                    {
                        if ((IsisSystem.Debug & (IsisSystem.MESSAGELAYER | IsisSystem.DISCARDS)) != 0)
                            Isis.WriteLine("Discarding an incoming CAST: I'm not a member of group " + gaddr + ", message was " + m);
                        using (new LockAndElevate(IsisSystem.RTS.Lock))
                            IsisSystem.RTS.Discarded++;
                        return false;
                    }

                case Msg.RUNASWORKER:
                    P2PSequencer.P2PCBWhenReady(m.sender, m, null);
                    if (IsisSystem.MasterCallBack == null)
                        Isis.WriteLine("Received RunAsWorker() request but RegisterAsMaster() wasn't called first");
                    else if (!knownWorkers.Contains(sender))
                    {
                        knownWorkers.Add(sender);
                        IsisSystem.MasterCallBack(sender);
                    }
                    break;
            }
            return true;
        }

        internal static byte[] byteConcatenate(byte[] a, byte[] b)
        {
            byte[] concatenated = new byte[a.Length + b.Length];
            concatenated = byteConcatenate(a, b);
            int off = 0;
            for (int i = 0; i < a.Length; i++)
                concatenated[off++] = a[i];
            for (int i = 0; i < a.Length; i++)
                concatenated[off++] = b[i];
            return concatenated;
        }

        internal static void SendAck(Group g, Address dest, int AckID)
        {
            if (g != null)
            {
                if ((g.flags & Group.G_ISLARGE) != 0 && g.theView != null && g.theView.members.Length > Group.tokenInfo.RINGSIZE)
                {
                    if ((IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0)
                        Isis.WriteLine("SendAck: Inhibiting ack: Group is large");
                    return;
                }
                else
                    g.interesting = true;
            }
            int minUID = AckID, maxUID = AckID;
            List<int> oamList = new List<int>();
            oamList.Add(AckID);
            List<Group> igc = Group.IsisGroupsClone();
            foreach (Group gr in igc)
            {
                if ((gr.flags & Group.G_ISLARGE) != 0)
                    continue;
                using (new LockAndElevate(gr.UnstableLock))
                    foreach (Msg m in gr.Unstable)
                        if (m.sender.Equals(dest))
                        {
                            minUID = Math.Min(minUID, m.UID);
                            maxUID = Math.Max(maxUID, m.UID);
                            oamList.Add(m.UID);
                        }
            }
            bool[] acked = new bool[Math.Min(Isis.ISIS_MAXMSGLEN * 8 / 2, maxUID - minUID + 1)];
            foreach (int uid in oamList)
                if ((uid - minUID) < acked.Length)
                    acked[uid - minUID] = true;
            if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
            {
                string ids = " ";
                foreach (int uid in oamList)
                    ids += uid + " ";
                using (new LockAndElevate(ackInfoLock))
                    ackInfo.Add("Sending ack: to " + dest + " ids = {" + ids + "}");
            }
            try
            {
                if ((IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0)
                    Isis.WriteLine("Sending an Ack: dest=" + dest + ", AckID=" + AckID);
                using (new LockAndElevate(IsisSystem.RTS.Lock))
                    IsisSystem.RTS.ACKsent++;
                byte[] b = Msg.toBArray(Isis.ISIS_HDR, Isis.my_address, minUID, acked);
                IPEndPoint remoteEP = new IPEndPoint(dest.home, dest.ackPort);
                if (Isis.ISIS_TCP_ONLY)
                    TCPSendTo(dest, b, remoteEP, ACKBB);
                else
                {
                    ReliableSender.CheckLen(b);
                    my_acksocket.SendTo(b, remoteEP);
                }
            }
            catch (ObjectDisposedException) { }
            catch (SocketException) { }
        }

        internal class SentNack
        {
            internal long when;
            internal Address SentTo;
            internal Address MsgGroup;
            internal Address MsgSender;
            internal int MsgVid;
            internal int MsgMsgid;

            internal SentNack(Address SentTo, Address MsgGroup, Address MsgSender, int MsgVid, int MsgMsgid)
            {
                this.when = Isis.NOW();
                this.SentTo = SentTo;
                this.MsgGroup = MsgGroup;
                this.MsgSender = MsgSender;
                this.MsgVid = MsgVid;
                this.MsgMsgid = MsgMsgid;
            }
        }

        internal class SentP2PNack
        {
            internal long when;
            internal Address SentTo;
            internal int p2pseqn;

            internal SentP2PNack(Address SentTo, int p2pseqn)
            {
                this.when = Isis.NOW();
                this.SentTo = SentTo;
                this.p2pseqn = p2pseqn;
            }
        }

        internal static List<SentNack> SentNacksTo = new List<SentNack>();
        internal static List<SentP2PNack> SentP2PNacksTo = new List<SentP2PNack>();
        internal static LockObject SentNacksLock = new LockObject("SendNacksLock", ThreadPriority.Highest);

        internal static void SendNack(Group g, Address SendTo, Address MsgSender, int MsgVid, int MsgMsgid)
        {
            if (g == null)
                return;
            using (new LockAndElevate(Isis.RIPLock))
                if (Isis.RIPList.Contains(SendTo))
                    return;
            using (new LockAndElevate(SentNacksLock))
            {
                foreach (SentNack sn in SentNacksTo)
                    if (sn.MsgGroup.Equals(g.gaddr) && sn.SentTo.Equals(SendTo) && sn.MsgSender.Equals(MsgSender) && sn.MsgVid == MsgVid && sn.MsgMsgid == MsgMsgid)
                        return;
                SentNacksTo.Add(new SentNack(SendTo, g.gaddr, MsgSender, MsgVid, MsgMsgid));
            }
            if ((IsisSystem.Debug & IsisSystem.NACKS) != 0)
                Isis.WriteLine("Sending a Nack: Group <" + g.gname + ">, sending a retransmit request to " + SendTo + " for Msg from sender=" + MsgSender + ", id " + MsgVid + ":" + MsgMsgid);
            if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                using (new LockAndElevate(ackInfoLock))
                    ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: Send NACK to " + SendTo + " nacking " + Isis.my_address + "::" + MsgVid + ":" + MsgMsgid + "\r\n");
            byte[] b = Msg.toBArray(Isis.ISIS_HDR, Isis.my_address, g.gaddr, MsgSender, MsgVid, MsgMsgid);
            using (new LockAndElevate(IsisSystem.RTS.Lock))
                IsisSystem.RTS.NACKsent++;
            P2PSend(SendTo, SendTo.ackPort, b, ACKBB);
        }

        internal static void SendP2PNack(Address sender, int p2pseqn)
        {
            using (new LockAndElevate(Isis.RIPLock))
                if (Isis.RIPList.Contains(sender))
                    return;
            using (new LockAndElevate(SentNacksLock))
            {
                foreach (SentP2PNack sn in SentP2PNacksTo)
                    if (sn.SentTo.Equals(sender) && sn.p2pseqn == p2pseqn)
                        return;
                SentP2PNacksTo.Add(new SentP2PNack(sender, p2pseqn));
            }
            if ((IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0)
                Isis.WriteLine("Sending a P2P Nack to " + sender + " for Msg with p2p seqn=" + p2pseqn);
            if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                using (new LockAndElevate(ackInfoLock))
                    ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: Send P2PNACK to " + sender + " nacking " + p2pseqn + "\r\n");
            byte[] b = Msg.toBArray(Isis.ISIS_HDR, Isis.my_address, sender, p2pseqn);
            P2PSend(sender, sender.ackPort, b, ACKBB);
        }

        internal static void CleanSentNack()
        {
            using (new LockAndElevate(SentNacksLock))
            {
                List<SentNack> newSNList = new List<SentNack>();
                foreach (SentNack sn in SentNacksTo)
                    if ((Isis.NOW() - sn.when) < 1500)
                        newSNList.Add(sn);
                SentNacksTo = newSNList;
                List<SentP2PNack> newP2PSNList = new List<SentP2PNack>();
                foreach (SentP2PNack sn in SentP2PNacksTo)
                    if ((Isis.NOW() - sn.when) < 1500)
                        newP2PSNList.Add(sn);
                SentP2PNacksTo = newP2PSNList;
            }
        }

        internal static void SendPoison(Address dest, string why)
        {
            if (!IsisSystem.IsisActive)
                throw new IsisShutdown("Isis inactive");
            if ((IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0)
                Isis.WriteLine("Sending POISON: dest=" + dest + ", reason=" + why);
            byte[] b = Msg.toBArray(Isis.ISIS_HDR, Isis.NULLADDRESS, -1, why);
            P2PSend(dest, dest.ackPort, b, ACKBB);
            using (new LockAndElevate(HeardFromLock))
                if (HeardFrom.ContainsKey(dest))
                    HeardFrom.Remove(dest);
        }

        delegate void ADel(byte[] hdr, Address sender, int AckID);

        internal static BoundedBuffer AckBB;

        internal class AckInfoItem
        {
            internal Address who;
            internal int id;

            internal AckInfoItem(Address s, int uid) { who = s; id = uid; }
        }
        internal class NackInfoItem
        {
            internal Address from;
            internal Address gaddr;
            internal Address msgsender;
            internal int viewid;
            internal int msgid;

            internal NackInfoItem(Address s, Address ga, Address ms, int vid, int mid) { from = s; gaddr = ga; msgsender = ms; viewid = vid; msgid = mid; }
        }

        internal static Dictionary<Address, SlidingBitBucket> ackHash = new Dictionary<Address, SlidingBitBucket>(5000);
        internal static LockObject ackHashLock = new LockObject("ackHashLock", ThreadPriority.Highest);

        internal static void AckNoteFailure(Address who)
        {
            using (new LockAndElevate(ackHashLock))
                if (ackHash.ContainsKey(who))
                    ackHash.Remove(who);
        }

        internal static void AckSocketReader(object os)
        {
            while (!IsisSystem.IsisActive)
                Thread.Sleep(250);
            if (!Isis.ISIS_TCP_ONLY)
                AckBB = new BoundedBuffer("BB:Ack", 512, ILock.LLACKB, 0, 1);
            Socket s = (Socket)os;
            if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                using (new LockAndElevate(ackInfoLock))
                    ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: {Creating ack thread at time " + Isis.TimeToString(Isis.NOW()) + "}\r\n");
            Thread ackProcessingThread = new Thread(delegate()
            {
                int len = 0;
                try
                {
                    if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                        using (new LockAndElevate(ackInfoLock))
                            ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: {Ack thread running at time " + Isis.TimeToString(Isis.NOW()) + "}\r\n");
                    while (IsisSystem.IsisActive)
                    {
                        IsisSystem.RTS.ThreadCntrs[23]++;
                        byte[] b = null;
                        try
                        {
                            List<AckInfoItem> p2pNackList = new List<AckInfoItem>();
                            List<NackInfoItem> nackList = new List<NackInfoItem>();
                            int lo = 0, hi = 0;
                            int cnt;
                            using (new LockAndElevate(AckBB.Lock))
                                cnt = Math.Min(Math.Max(1, AckBB.fullSlots), 256);
                            while (cnt-- > 0)
                            {
                                object next = AckBB.get();
                                if (next == null)
                                    break;
                                b = (byte[])next;
                                len = b.Length;
                                using (new LockAndElevate(PendingSendBufferLock))
                                {
                                    if (PendingSendBuffer.Count() > 0)
                                    {
                                        lo = Math.Min(lo, PendingSendBuffer.First.Value.UID);
                                        hi = Math.Max(hi, PendingSendBuffer.Last.Value.UID);
                                    }
                                }
                                object[] obs;
                                try
                                {
                                    obs = Msg.BArrayToObjects(b);
                                }
                                catch
                                {
                                    continue;
                                }
                                int idx = 0;
                                if (Msg.CheckTypes(obs, typeof(byte[]), typeof(Address), typeof(int), typeof(string)))
                                {
                                    // Poison
                                    byte[] hdr = (byte[])obs[idx++];
                                    Address sender = (Address)obs[idx++];
                                    int AckID = (int)obs[idx++];
                                    if (!chkhdr(hdr) || !sender.Equals(Isis.NULLADDRESS) || AckID != -1)
                                    {
                                        if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                                            Isis.WriteLine("Ignoring a malformed POISON message");
                                        using (new LockAndElevate(IsisSystem.RTS.Lock))
                                            IsisSystem.RTS.Discarded++;
                                    }
                                    else
                                    {
                                        string why = (string)obs[idx++];
                                        IsisSystem.GotPoison(why);
                                    }
                                }
                                if (Msg.CheckTypes(obs, typeof(byte[]), typeof(Address), typeof(int), typeof(bool[])))
                                {
                                    // Normal Ack
                                    byte[] hdr = (byte[])obs[idx++];
                                    if (!chkhdr(hdr))
                                        continue;
                                    Address sender = (Address)obs[idx++];
                                    int minUID = (int)obs[idx++];
                                    bool[] ackedIds = (bool[])obs[idx++];
                                    justHeardFrom(sender);
                                    if (sender.Equals(Isis.NULLADDRESS))
                                        IsisSystem.GotPoison("(unsure)");
                                    SlidingBitBucket sbb;
                                    using (new LockAndElevate(ackHashLock))
                                        if (ackHash.TryGetValue(sender, out sbb) == false)
                                            ackHash.Add(sender, sbb = new SlidingBitBucket(lo, hi));
                                    if (minUID < lo)
                                    {
                                        using (new LockAndElevate(PendingSendBufferLock))
                                            if (PendingSendBuffer.Count() > 0 && PendingSendBuffer.First.Value.UID < lo)
                                                throw new IsisException("minUID was " + minUID + " and psb.First=" + PendingSendBuffer.First.Value.UID + ", but lo=" + lo);
                                    }
                                    sbb.reset(lo, hi);
                                    for (int bit = 0; bit < ackedIds.Length; bit++)
                                        if (ackedIds[bit])
                                            sbb.Set(bit + minUID);
                                    if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                                    {
                                        string ids = " ";
                                        for (int bit = 0; bit < ackedIds.Length; bit++)
                                            if (ackedIds[bit])
                                                ids += (bit + minUID) + " ";
                                        using (new LockAndElevate(ackInfoLock))
                                            ackInfo.Add("Ack from " + sender + " ids = {" + ids + "}");
                                        ids = " ";
                                        using (new LockAndElevate(PendingSendBufferLock))
                                            foreach (MsgDesc md in PendingSendBuffer)
                                                if (md.dest.Equals(sender))
                                                    ids += md.UID + " ";
                                        using (new LockAndElevate(ackInfoLock))
                                            ackInfo.Add("... need: " + sender + " ids = {" + ids + "}");
                                    }
                                    using (new LockAndElevate(IsisSystem.RTS.Lock))
                                        ++IsisSystem.RTS.ACKrcvd;
                                }
                                else if (Msg.CheckTypes(obs, typeof(byte[]), typeof(Address), typeof(Address), typeof(Address), typeof(int), typeof(int)))
                                {
                                    // Nack for a missing packet
                                    byte[] hdr = (byte[])obs[idx++];
                                    if (!chkhdr(hdr))
                                        continue;
                                    Address from = (Address)obs[idx++];
                                    Address gaddr = (Address)obs[idx++];
                                    Address msgsender = (Address)obs[idx++];
                                    int viewid = (int)obs[idx++];
                                    int msgid = (int)obs[idx++];
                                    bool fnd = false;
                                    foreach (NackInfoItem nai in nackList)
                                        if (nai.from.Equals(from) && nai.gaddr.Equals(gaddr) && nai.msgsender.Equals(msgsender) && nai.viewid == viewid && nai.msgid == msgid)
                                            fnd = true;
                                    if (!fnd)
                                        nackList.Add(new NackInfoItem(from, gaddr, msgsender, viewid, msgid));
                                    using (new LockAndElevate(IsisSystem.RTS.Lock))
                                        ++IsisSystem.RTS.NACKrcvd;
                                }
                                else if (Msg.CheckTypes(obs, typeof(byte[]), typeof(Address), typeof(Address), typeof(int)))
                                {
                                    // Nack for a missing P2P packet
                                    byte[] hdr = (byte[])obs[idx++];
                                    if (!chkhdr(hdr))
                                        continue;
                                    Address requestedBy = (Address)obs[idx++];
                                    Address originalSender = (Address)obs[idx++];
                                    if (!originalSender.isMyAddress())
                                        throw new IsisException("Unexpectedly received NACK for a p2p packet originating at " + originalSender);
                                    int p2pseqn = (int)obs[idx];
                                    bool fnd = false;
                                    foreach (AckInfoItem p2pi in p2pNackList)
                                        if (p2pi.who.Equals(requestedBy) && p2pi.id == p2pseqn)
                                        {
                                            fnd = true;
                                            break;
                                        }
                                    if (!fnd)
                                        p2pNackList.Add(new AckInfoItem(requestedBy, p2pseqn));
                                    using (new LockAndElevate(IsisSystem.RTS.Lock))
                                        ++IsisSystem.RTS.ACKrcvd;
                                }
                                else
                                {
                                    if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                                    {
                                        string tsig = " ";
                                        foreach (object o in obs)
                                            tsig += o.GetType() + " ";
                                        Isis.WriteLine("WARNING: Discarding an ack/nack because its type signature didn't match any known signature!  TS={" + tsig + "}");
                                    }
                                    using (new LockAndElevate(IsisSystem.RTS.Lock))
                                        IsisSystem.RTS.Discarded++;
                                }
                            }
                            if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                                using (new LockAndElevate(IsisSystem.RTS.Lock))
                                {
                                    string acks = " ";
                                    using (new LockAndElevate(ackHashLock))
                                        foreach (KeyValuePair<Address, SlidingBitBucket> kvp in ackHash)
                                            for (int uid = lo + 1; uid <= hi; uid++)
                                                if (kvp.Value.Test(uid))
                                                    acks += kvp.Key + "::" + uid + " ";
                                    using (new LockAndElevate(ackInfoLock))
                                        ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: Got acks for " + acks + "\r\n");
                                }
                            using (new LockAndElevate(IsisSystem.RTS.Lock))
                                IsisSystem.RTS.ackProcessingBeganAt = Isis.NOW();
                            // Acks are common, so process as a list.  Nacks and P2P nacks are rare and often an Ack renders the action unneeded by the time we see the Nack
                            using (new LockAndElevate(ackHashLock))
                                GotAnAck(lo, hi, ackHash);
                            foreach (AckInfoItem p2pi in p2pNackList)
                                GotP2PNack(p2pi.who, p2pi.id);
                            foreach (NackInfoItem nacki in nackList)
                                GotNack(nacki.from, nacki.gaddr, nacki.msgsender, nacki.viewid, nacki.msgid);
                            using (new LockAndElevate(IsisSystem.RTS.Lock))
                                IsisSystem.RTS.ackProcessingBeganAt = 0;
                        }
                        catch (Exception e)
                        {
                            try
                            {
                                object[] obs = Msg.BArrayToObjects(b);
                                string ts = "";
                                for (int o = 0; o < obs.Length; o++)
                                    ts += " " + obs[o].GetType() + " ";
                                Isis.WriteLine("AckSocketReader threw exception " + e + " on Isis Message with type signature {" + ts + "}... IGNORING IT");
                            }
                            catch (Exception ie)
                            {
                                IsisSystem.Shutdown("AckSocketReader threw exception " + e + " then inner " + ie + ": len was " + len);
                            }
                        }
                    }
                }
                catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                if (IsisSystem.IsisActive)
                    throw new IsisException("ackProcessingThread has failed");
                IsisSystem.ThreadTerminationMagic();
            });
            ackProcessingThread.Name = "Ack processing thread";
            ackProcessingThread.Priority = ThreadPriority.Highest;
            ackProcessingThread.Start();
            if (!Isis.ISIS_TCP_ONLY)
            {
                try
                {
                    while (!IsisSystem.IsisActive)
                        Thread.Sleep(250);
                    Thread.CurrentThread.Priority = ThreadPriority.Highest;
                    while (IsisSystem.IsisActive)
                    {
                        IsisSystem.RTS.ThreadCntrs[24]++;
                        byte[] b = new byte[Isis.ISIS_MAXMSGLEN + 1024];
                        int rlen;
                        do
                        {
                            try
                            {
                                rlen = s.Receive(b);
                            }
                            catch (IOException) { rlen = 0; }
                            catch (SocketException) { rlen = 0; }
                            IsisSystem.RTS.ThreadCntrs[0]++;
                        }
                        while (!Msg.VerifySignature(b, 0, rlen));
                        IsisSystem.RTS.ThreadCntrs[1]++;

                        if (rlen > Isis.ISIS_MAXMSGLEN)
                            Isis.WriteLine("WARNING (b): MAXMSGLEN set to " + Isis.ISIS_MAXMSGLEN + ", but received length=" + rlen);
                        if (rlen > 4)
                        {
                            Isis.ArrayResize<byte>(ref b, rlen);
                            using (new LockAndElevate(IsisSystem.RTS.Lock))
                            {
                                IsisSystem.RTS.ackedAt = Isis.NOW();
                                if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                                    using (new LockAndElevate(ackInfoLock))
                                        ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: Receive(ack-socket)\r\n");
                            }
                            AckBB.put(b, true);
                            IsisSystem.RTS.check();
                            using (new LockAndElevate(IsisSystem.RTS.Lock))
                                IsisSystem.RTS.ackedAt = 0;
                        }
                    }
                }
                catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                if (IsisSystem.IsisActive)
                    throw new IsisException("ack receiver thread has failed");
            }
            else
                throw new ThreadInterruptedException();
        }

        private static bool chkhdr(byte[] hdr)
        {
            if (hdr.Length != Isis.ISIS_HDR.Length)
                goto noMatch;
            for (int i = 0; i < hdr.Length; i++)
                if (hdr[i] != Isis.ISIS_HDR[i])
                    goto noMatch;
            return true;
        noMatch:
            if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                Isis.WriteLine("WARNING: discarding an ack/nack because it didn't have the correct Isis header");
            using (new LockAndElevate(IsisSystem.RTS.Lock))
                IsisSystem.RTS.Discarded++;
            return false;
        }

        internal static Dictionary<Address, long> HeardFrom = new Dictionary<Address, long>(100);
        internal static LockObject HeardFromLock = new LockObject("HeardFromLock", ThreadPriority.Highest);

        internal static string HeardFromState()
        {
            string s = "Heard from recently: {";
            using (new LockAndElevate(HeardFromLock))
                foreach (KeyValuePair<Address, long> kvp in HeardFrom)
                {
                    string t;
                    if (kvp.Value > Isis.NOW())
                        t = "<GRACE PERIOD>";
                    else
                        t = Isis.TimeToString(kvp.Value);
                    s += kvp.Key.ToString() + "@" + t + " ";
                }
            return s + "}\r\n";
        }

        internal static bool HeardFromRecently(Address who)
        {
            long when;
            // During startup be slow to detect failures....
            if (Isis.NOW() < 30000)
                return true;
            using (new LockAndElevate(HeardFromLock))
                HeardFrom.TryGetValue(who, out when);
            return ((Isis.NOW() - when) < Isis.ISIS_DEFAULTTIMEOUT);
        }

        internal static void justHeardFrom(Address who)
        {
            using (new LockAndElevate(HeardFromLock))
            {
                if (HeardFrom.ContainsKey(who))
                    HeardFrom.Remove(who);
                HeardFrom.Add(who, Isis.NOW());
            }
        }

        internal static void nodeInStartup(Address who)
        {
            // Special case: give 20 seconds of grace, because C# class loader can lock out I/O during the startup period
            using (new LockAndElevate(HeardFromLock))
                HeardFrom[who] = Isis.NOW() + Isis.GRACEPERIOD;
        }

        internal class SIDPair
        {
            internal Address Who;
            internal int ID;

            internal SIDPair(Address w, int i) { Who = w; ID = i; }
        }

        internal class SlidingBitBucket
        {
            internal int lo;
            internal int hi = 511;
            internal bool[] bits = new bool[512];

            internal SlidingBitBucket(int newHi)
            {
                lo = 0;
                hi = newHi;
                for (int bit = lo; bit <= newHi; bit++)
                    bits[bit] = true;
            }

            internal SlidingBitBucket(int initLo, int initHi)
            {
                reset(initLo, initHi);
            }

            internal void reset(int newLo, int newHi)
            {
                lo = Math.Max(newLo, lo);
                if (hi >= newHi)
                    return;
                int n = bits.Length;
                if (newHi - lo + 1 > n)
                    Array.Resize<bool>(ref bits, n = ((newHi - lo) / 64 + 1) * 64);
                while (hi < newHi)
                    bits[++hi % n] = false;
                hi = newHi;
            }

            internal void Set(int bit)
            {
                if (bit > hi)
                    reset(lo, bit);
                if (bit < lo || bit > hi)
                    return;
                bits[bit % bits.Length] = true;
            }

            internal bool Test(int bit)
            {
                if (bit < lo || bit > hi)
                    return false;
                return bits[bit % bits.Length];
            }
        }

        internal static List<string> ackInfo = ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0) ? new List<string>() : null;
        internal static LockObject ackInfoLock = new LockObject("ackInfoLock");
        internal static List<object> toResend = new List<object>();
        internal static Semaphore toResendSema = new Semaphore(0, int.MaxValue);
        internal static LockObject toResendLock = new LockObject("toResendLock");
        internal static bool resenderRunning = false;
        internal static Thread ackNackResenderThread = null;
        internal static LockObject AckLock = new LockObject("AckLock", ThreadPriority.Highest);
        internal static LockObject SingletonLock = new LockObject("SingletonLock", ThreadPriority.Highest);
        internal static SlidingBitBucket sbb = new SlidingBitBucket(0);
        internal static Dictionary<Address, SlidingBitBucket> singleton = new Dictionary<Address, SlidingBitBucket>(1);

        internal static void GotAnAck(Address sender, int AckID)
        {
            using (new LockAndElevate(SingletonLock))
            {
                singleton.Add(sender, sbb);
                GotAnAck(AckID - 1, AckID, singleton);
                singleton.Remove(sender);
            }
        }

        internal static void GotAnAck(int lo, int hi, Dictionary<Address, SlidingBitBucket> alist)
        {
            using (new LockAndElevate(AckLock))
            {
                List<MsgDesc> toRemove = new List<MsgDesc>();
                LinkedList<MsgDesc> newPSB = new LinkedList<MsgDesc>();
                using (new LockAndElevate(PendingSendBufferLock))
                {
                    SlidingBitBucket sbb;
                    foreach (MsgDesc md in PendingSendBuffer)
                        if (md.UID >= lo && md.UID <= hi && alist.TryGetValue(md.dest, out sbb) && sbb.Test(md.UID))
                            toRemove.Add(md);
                        else
                            newPSB.AddLast(md);
                    PendingSendBuffer = newPSB;
                }
                if (toRemove.Count() > 0)
                {
                    foreach (MsgDesc md in toRemove)
                        if (md.group != null)
                        {
                            bool wantsStabilitySent;
                            using (new LockAndElevate(md.group.GroupFlagsLock))
                                wantsStabilitySent = md.group.SendStabilityNeeded;
                            if (md.localSender && !wantsStabilitySent && md.type != Msg.STABILITYINFO)
                            {
                                using (new LockAndElevate(PendingSendBufferLock))
                                    wantsStabilitySent = md.group.multicastSendIsStable(md.MsgVid, md.MsgId);
                                using (new LockAndElevate(md.group.GroupFlagsLock))
                                    md.group.SendStabilityNeeded = wantsStabilitySent;
                            }
                        }
                    int cnt = toRemove.Count();
                    foreach (MsgDesc md in toRemove)
                    {
                        md.done = true;
                        FlushingRemove(md, "md.GotAnAck", --cnt == 0);
                    }
                }
            }
            TickleResenderThread();
        }

        internal static void GotNack(Address from, Address gaddr, Address MsgSender, int MsgVid, int MsgMsgid)
        {
            bool launchThread = false;
            using (new LockAndElevate(toResendLock))
                if (!resenderRunning)
                {
                    resenderRunning = true;
                    launchThread = true;
                }
            if (launchThread)
            {
                ackNackResenderThread = new Thread(delegate()
                {
                    try
                    {
                        while (!IsisSystem.IsisActive)
                            Thread.Sleep(250);
                        while (IsisSystem.IsisActive)
                        {
                            IsisSystem.RTS.ThreadCntrs[25]++;
                            object o;
                            toResendSema.WaitOne();
                            using (new LockAndElevate(toResendLock))
                            {
                                if (toResend.Count() == 0)
                                    if (!IsisSystem.IsisActive)
                                        goto done;
                                    else
                                        throw new IsisException("toResend shouldn't be empty");
                                o = toResend.First();
                                toResend.Remove(o);
                            }
                            if (o.GetType().Equals(typeof(MsgDesc)))
                            {
                                MsgDesc md = (MsgDesc)o;
                                if (md.done)
                                    continue;
                                if (md.isp2p)
                                {
                                    if (md.dest.isMyAddress())
                                        Isis.WriteLine("ack/nack resender: sending to self");
                                    if (Isis.ISIS_TCP_ONLY)
                                    {
                                        if (Isis.ISISMEMBERS != null && Isis.ISISMEMBERS.HasFirstView)
                                            Isis.ISISMEMBERS.UDPTunnel(md.dest, md.retryCnt, md.theBytes, 0);
                                        else if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                                            Isis.WriteLine("WARNING: Unable to respond to a NACK because ISISMEMBERS isn't intialized yet.");
                                    }
                                    else
                                    {
                                        IPEndPoint target = new IPEndPoint(md.dest.home, md.dest.p2pPort);
                                        ReliableSender.CheckLen(md.theBytes);
                                        int bs = my_p2psocket.SendTo(md.theBytes, target);
                                        using (new LockAndElevate(IsisSystem.RTS.Lock))
                                        {
                                            IsisSystem.RTS.UDPBsent += bs;
                                            IsisSystem.RTS.UDPsent++;
                                        }
                                    }
                                }
                                else if (md.isLarge)
                                    ReMulticast(md);
                                else if (Resend(md))
                                    ++md.retryCnt;
                            }
                            else if (o.GetType().Equals(typeof(Msg)))
                            {
                                Msg m = (Msg)o;
                                Group theGroup = Group.doLookup(m.gaddr);
                                if (theGroup != null)
                                    ReliableSender.SendP2P(m.type, from, theGroup, ResendForm(m), false);
                            }
                        }
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                done:
                    IsisSystem.ThreadTerminationMagic();
                });
                ackNackResenderThread.Name = "Ack/Nack resender thread";
                ackNackResenderThread.Priority = ThreadPriority.Highest;
                ackNackResenderThread.Start();
            }
            using (new LockAndElevate(IsisSystem.RTS.Lock))
                IsisSystem.RTS.NACKrcvd++;
            Group g = Group.doLookup(gaddr);
            if (g == null)
                return;
            using (new LockAndElevate(AckLock))
            {
                if ((IsisSystem.Debug & IsisSystem.NACKS) != 0)
                    Isis.WriteLine("Received a NACK from " + from + " for group <" + (g == null ? "null" : g.gname) + ">, for Msg from Sender=" + MsgSender + " with msgid=" + MsgVid + ":" + MsgMsgid);
                if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                    using (new LockAndElevate(ackInfoLock))
                        ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: Got nack from " + from + " requesting " + MsgSender + "::" + MsgVid + ":" + MsgMsgid + "\r\n");

                if ((g.flags & Group.G_ISLARGE) == 0)
                {
                    MsgDesc mdfnd = null;
                    if (MsgSender.isMyAddress())
                    {
                        using (new LockAndElevate(ReliableSender.PendingSendBufferLock))
                        {
                            foreach (MsgDesc md in PendingSendBuffer)
                                if (md.dest.Equals(from) && md.MsgVid == MsgVid && md.MsgId == MsgMsgid)
                                {
                                    mdfnd = md;
                                    break;
                                }
                        }
                        if (mdfnd != null)
                        {
                            using (new LockAndElevate(toResendLock))
                                if (!toResend.Contains(mdfnd))
                                {
                                    toResend.Add(mdfnd);
                                    toResendSema.Release(1);
                                }
                            return;
                        }
                    }
                    List<Msg> resendList = new List<Msg>();
                    using (new LockAndElevate(g.UnstableLock))
                        foreach (Msg m in g.Unstable)
                            if (m.sender.Equals(MsgSender) && ((g.flags & Group.G_ISLARGE) != 0 || m.vid == MsgVid) && m.msgid == MsgMsgid)
                                resendList.Add(m);
                    if (resendList.Count() == 0)
                    {
                        if ((IsisSystem.Debug & IsisSystem.NACKS) != 0)
                            Isis.WriteLine("WARNING: Didn't find NACKED packet on the unstable list!");
                        return;
                    }
                    foreach (Msg mtrs in resendList)
                    {
                        mtrs.gaddr = g.gaddr;
                        using (new LockAndElevate(toResendLock))
                        {
                            if (!toResend.Contains(mtrs))
                            {
                                toResend.Add(mtrs);
                                toResendSema.Release(1);
                            }
                        }
                    }
                    return;
                }
                // Large group case: Messages identified ONLY by msgid here (vid may have a nonsense value) and will be found on the LgPendingSendBuffer list
                MsgDesc theLgMd = null;
                using (new LockAndElevate(ReliableSender.PendingSendBufferLock))
                {
                    foreach (MsgDesc lgmd in LgPendingSendBuffer)
                        if (lgmd.MsgId == MsgMsgid)
                        {
                            theLgMd = lgmd;
                            break;
                        }
                }
                if (theLgMd != null)
                {
                    using (new LockAndElevate(toResendLock))
                    {
                        if (!toResend.Contains(theLgMd))
                        {
                            toResend.Add(theLgMd);
                            toResendSema.Release(1);
                        }
                    }
                }
                else if ((IsisSystem.Debug & IsisSystem.NACKS) != 0)
                    Isis.WriteLine("WARNING: Didn't find NACKED packet on the LgPendingSendBuffer list!");
            }
            TickleResenderThread();
        }

        internal static Msg ResendForm(Msg m)
        {
            return m.asReceived ?? m;
        }

        internal static void GotP2PNack(Address requestedBy, int p2pseqn)
        {
            using (new LockAndElevate(AckLock))
            {
                MsgDesc md = null;
                string rsi = " ";
                using (new LockAndElevate(PendingSendBufferLock))
                    foreach (MsgDesc psmd in PendingSendBuffer)
                        if (psmd.dest.Equals(requestedBy) && psmd.UID == p2pseqn)
                        {
                            md = psmd;
                            break;
                        }
                        else
                            rsi += "<" + psmd.dest + ":" + psmd.UID + ">";
                if ((IsisSystem.Debug & IsisSystem.LOWLEVELMSGS) != 0)
                    using (new LockAndElevate(ackInfoLock))
                        ackInfo.Add("[" + Isis.TimeToString(Isis.NOW()) + "]: Received a P2P NACK from " + requestedBy + " for p2pSeqn = " + p2pseqn + "\r\n");
                if ((IsisSystem.Debug & IsisSystem.NACKS) != 0)
                    if (md == null)
                        Isis.WriteLine("WARNING: Received a P2P NACK from " + requestedBy + " but didn't find packet with p2pseqn=" + p2pseqn);
                    else
                        Isis.WriteLine("Received a P2P NACK from " + requestedBy + " for message with p2pseqn=" + p2pseqn + "(msg id " + md.MsgVid + ":" + md.MsgId + "), found it and will resend");
                if (md == null && genFiller(requestedBy, p2pseqn))
                {
                    if ((IsisSystem.Debug & IsisSystem.NACKS) != 0)
                        Isis.WriteLine("Generating Msg.ISFILLER for " + requestedBy + ", seqn=" + p2pseqn);
                    md = new MsgDesc(Msg.ISFILLER, 1, null, requestedBy, true, false, p2pseqn,
                                        Msg.toBArray(Msg.ISFILLER, (byte)0, p2pseqn, Isis.my_address, requestedBy, Isis.NULLADDRESS, 0, 0, Msg.NewMsgAsBArray(Isis.my_address, Isis.NULLADDRESS, 0, p2pseqn, 0L, 0, p2pseqn, 0, new byte[0])),
                                        0, 0, 0, 0, 1, true, null, null);
                }
                if (md != null)
                    using (new LockAndElevate(toResendLock))
                    {
                        if (!toResend.Contains(md))
                        {
                            toResend.Add(md);
                            toResendSema.Release(1);
                        }
                    }
            }
            TickleResenderThread();
        }

        /*
         * This code is capable of generating a "fake" retransmission of a message that is, in fact, completely missing.
         * It isn't intended for real use but can be a useful tool when debugging certain kinds of low-level problems in the system.
         * Currently commented out because technically speaking, filling a gap with a no-op message this way is a mistake!
         * */
        internal static bool genFiller(Address who, int seqn)
        {
            /*
            using (new LockAndElevate(P2PSequencer.PSListLock))
            {
                foreach (P2PSequencer p2ps in P2PSequencer.PSList)
                    if (p2ps.dest.Equals(who) && p2ps.remoteBacklog > 100 && p2ps.outSeqn-seqn > 100)
                        return true;
            }
             * */
            return false;
        }

        // Because zero-bytes are so common, this does run-length encoding for runs of zeros (but not other values)
        // The true buffer length is stored at the very end of the compressed object
        internal static byte[] Compress(byte[] raw)
        {
            if (Isis.ISIS_DONT_COMPRESS == true || raw.Length < 128) return null;
            byte[] compressed = new byte[raw.Length];
            int idx = 0;
            for (int b = 0; b < raw.Length; b++)
            {
                if ((compressed[idx++] = raw[b]) == 0)
                {
                    ushort count = 0;
                    while (b < raw.Length - 1 && raw[b + 1] == 0 && count != 0x7FFF)
                    {
                        ++count; ++b;
                    }
                    if (count <= 0x7F)
                        compressed[idx++] = (byte)count;
                    else
                    {
                        compressed[idx++] = (byte)(((count >> 8) & 0x7F) | 0x80);
                        compressed[idx++] = (byte)(count & 0xFF);
                    }
                }
                if (idx > (raw.Length * 2) / 3) return null;
            }
            compressed[idx++] = (byte)((raw.Length >> 24) & 0xFF);
            compressed[idx++] = (byte)((raw.Length >> 16) & 0xFF);
            compressed[idx++] = (byte)((raw.Length >> 8) & 0xFF);
            compressed[idx++] = (byte)(raw.Length & 0xFF);
            Isis.ArrayResize<byte>(ref compressed, idx);
            return compressed;
        }

        internal static byte[] DeCompress(byte[] raw)
        {
            int expectedLen = ((int)raw[raw.Length - 4] << 24) + ((int)raw[raw.Length - 3] << 16) + ((int)raw[raw.Length - 2] << 8) + ((int)raw[raw.Length - 1]);
            byte[] decompressed = new byte[expectedLen];
            int idx = 0;
            for (int b = 0; b < raw.Length - 4; b++)
                if ((decompressed[idx++] = raw[b]) == 0)
                {
                    uint len = raw[++b];
                    if ((len & 0x80) != 0)
                        len = ((len & 0x7F) << 8) | raw[++b];
                    for (int a = 0; a < len; a++)
                        decompressed[idx++] = 0;
                }
            if (idx != expectedLen) throw new IsisException("Decompression failed");
            return decompressed;
        }
    }

    // I'm using this myself, but outside Isis users should only do P2P
    // communication in the context of some group for now.
    internal class P2P
    {
        internal static void Send(Address dest, params object[] obs)
        {
            byte[] buffer = Msg.toBArray(obs);
            if (dest.isMyAddress())
            {
                Msg m = new Msg();
                m.payload = buffer;
                m.sender = m.dest = dest;
                DeliverP2P(m);
                return;
            }
            ReliableSender.SendP2P(Msg.ISPUREP2P, dest, null, buffer, true);
        }

        internal static void DeliverP2P(Msg m)
        {
            if (!Isis.ORACLE.isTrackingProxy)
                Isis.ORACLE.enqueueForGroupDelivery(m);
            else // Special for OOB transfer of initial views
            {
                while (Isis.ISISMEMBERS == null)
                    Thread.Sleep(100);
                Isis.ISISMEMBERS.enqueueForGroupDelivery(m);
            }
        }
    }

    /// <summary>
    /// The Isis.Msg class defines the wire-representation of messages used by the Isis system.  
    /// </summary>
    /// <remarks>
    /// End users will normally not need
    /// direct access to this type of object, but the various C# scoping rules compell us to make it public, and there are static
    /// methods that the end user does need to call, for example to register a new data type that Isis will transport (<see cref="Msg.RegisterType"/>).
    /// </remarks>
    public class Msg : IComparable
    {
        /// <summary>
        /// Isis.Address of the sender of this message
        /// </summary>
        public Address sender;
        /// <summary>
        /// Isis.Address of the destination of this message
        /// </summary>
        public Address dest;
        /// <summary>
        /// Isis.Address of the group within which this message was sent
        /// </summary>
        internal Address gaddr;                       // Used only while on Unstable list

        /// <summary>
        /// Value of a viewid before Msg has a specified vid
        /// </summary>
        public const int UNINITIALIZED = -1;
        /// <summary>
        /// Viewid of the group at the time this message was sent
        /// </summary>
        public int vid = UNINITIALIZED;
        /// <summary>
        /// Sequence number unique to this sender for the message.  But other senders may use the same number
        /// for their own messages, so must treat as a pair: (msgid,sender-id) using sender-id to break ties.
        /// </summary>
        public int msgid = 0;
        /// <summary>
        /// Used for the logging ID, if the logging service is active
        /// </summary>
        public long Lid = 0;
        /// <summary>
        /// If we've been sending RAW messages, this gives a count of how many were
        /// sent prior to the current message.  Since those can be dropped in the network
        /// and won't be retransmitted, the receiver shouldn't delay this one if it follows a
        /// gap of size nRaw (or fewer) missing messages.  The algorithm is surprisingly subtle
        /// although it may look simple.  A further (major) complication arises because we treat
        /// subset multicasts (e.g. for DHTOrderedSend) as a form of raw multicast.  The multicast
        /// layer sees such messages as being entirely raw because we use a multicast msgid number but 
        /// in fact send the "multicast" purely via P2P messages in this case.  The P2P layer,
        /// however, encapsulates the subset multicast and sends it, reliably, using a P2P protocol
        /// over sequenced, reliable, P2P channels with nRaw=0 for the outer capsule messages.
        /// The idea is to "explain" to the multicast layer why it will see seemingly missing
        /// multicasts in the multicast sequence numbering.  Anyhow, all of this makes nRaw subtle.
        /// </summary>
        public int nRaw = 0;
        /// <summary>
        /// Set by the original sender, gives the UID by which the message should be acknowledged
        /// </summary>
        public int UID = 0;
        /// <summary>
        /// The body of the message, encoded into a byte array
        /// </summary>
        public byte[] payload;

        internal byte[] cipherPayload;   // Non-null if a cipher copy of the payload is available
        internal Msg asReceived;         // Non-null for the first fragment of a fragmented message so that p2p retransmission off unstable list will work
        internal byte[] offWire;         // As it came off the wire
        internal Group.CompletionTag ct; // Non-null for a safe message being delivered
        internal List<Address> destList; // Used to relay the List<Address> or QueryKey argument, if any, to doTheSend

        // Cached, for speed
        volatile internal object[] myObs;
        internal Address wasForwardedTo;
        internal bool ordered;
        internal long toDoTime = 0;
        internal LockObject Lock = new LockObject("Msg.Lock");

        internal byte type;                           // Value 0 (uninitialized) when message is on the wire or recently loaded
        // Type values
        internal const byte ISREPLY = 1;
        internal const byte FIFOCAST = 2;
        internal const byte UNORDERED = 3;
        internal const byte ISPUREP2P = 4;
        internal const byte ISGRPP2P = 5;
        internal const byte ISTOKEN = 6;
        internal const byte ISLOOPBACK = 7;
        internal const byte ISPING = 8;
        internal const byte ISPOISON = 9;
        internal const byte STABILITYINFO = 10;
        internal const byte RUNASWORKER = 11;
        internal const byte ISRAWGRPP2P = 12;
        internal const byte ISRAWREPLY = 13;
        internal const byte RAWFIFOCAST = 14;
        internal const byte REDELIVERY = 15;
        internal const byte ISFILLER = 16;

        internal static string[] mtypes = { "type unknown", "reply", "multicast", "unordered multicast", "pure p2p", "p2p in a group",
                                       "token", "loopback", "ping", "poison", "stability-info", "run as worker", 
                                       "raw (unreliable) p2p in a group", "raw (unreliable) reply", "raw (unreliable) multicast", "redelivery in orderedsend/safesend", "gap filler" };

        // Flag and associated values
        internal byte flags;
        internal const byte NEEDSREPLY = 0x01;
        internal const byte SENTBYORACLE = 0x02;
        internal const byte DEFRAGGED = 0x04;
        internal const byte HASREPLY = 0x08;
        internal const byte FRAGGED = 0x10;
        internal const byte TOKEN = 0x20;
        internal const byte CIPHER = 0x40;
        internal const byte ENCIPHEREDREPLY = 0x80;

        internal static string pflags(int f)
        {
            string fs = " ";
            if ((f & NEEDSREPLY) != 0)
                fs += "needs-reply ";
            if ((f & SENTBYORACLE) != 0)
                fs += "sent-by-oracle ";
            if ((f & DEFRAGGED) != 0)
                fs += "defragged ";
            if ((f & FRAGGED) != 0)
                fs += "fragged ";
            if ((f & TOKEN) != 0)
                fs += "token ";
            if ((f & CIPHER) != 0)
                fs += "enciphered ";
            if ((f & ENCIPHEREDREPLY) != 0)
                fs += "needs-enciphered-reply ";
            return fs;
        }

        /// <summary>
        /// Encodes an Isis.Msg object as a byte array.  Not normally needed by Isis end-users, but public to satisfy a C# scoping requirement.
        /// </summary>
        /// <returns>The byte[] representation of the given message, suitable for transmission</returns>
        public byte[] toBArray()
        {
            if ((flags & Msg.CIPHER) != 0)
                return Msg.NewMsgAsBArray(sender, dest, vid, msgid, Lid, nRaw, UID, flags, cipherPayload);
            return Msg.NewMsgAsBArray(sender, dest, vid, msgid, Lid, nRaw, UID, flags, payload);
        }

        /// <summary>
        /// Creates a new Isis.Msg object with a null payload.  Not normally needed by Isis end-users, but public to satisfy a C# scoping requirement.
        /// </summary>
        public Msg() { }

        /// <summary>
        /// Creates a new Isis.Msg object from a byte vector encoding a message
        /// </summary>
        /// <param name="ba">the byte[] representation of an encoded message</param>
        public Msg(byte[] ba)
        {
            object[] obs = BArrayToObjects(ba, typeof(Address), typeof(Address), typeof(int), typeof(int), typeof(long), typeof(int), typeof(int), typeof(byte), typeof(byte[]));
            int idx = 0;
            sender = (Address)obs[idx++];
            dest = (Address)obs[idx++];
            vid = (int)obs[idx++];
            msgid = (int)obs[idx++];
            Lid = (long)obs[idx++];
            nRaw = (int)obs[idx++];
            UID = (int)obs[idx++];
            flags = (byte)obs[idx++];
            if ((flags & CIPHER) == 0)
                payload = (byte[])obs[idx];
            else
                cipherPayload = (byte[])obs[idx];
        }

        /// <summary>
        /// Creates a new Isis.Msg object that encodes the given object vector.  Not normally needed by Isis end-users, but public to satisfy a C# scoping requirement.
        /// </summary>
        /// <param name="obs">The new message object</param>
        public Msg(params object[] obs)
        {
            sender = Isis.my_address;
            dest = Isis.NULLADDRESS;
            payload = toBArray(obs);
        }

        /// <summary>
        /// Converts an Isis.Msg object to a string representation for printing.  Not normally needed by Isis end-users, but must be public to satisfy a C# scoping requirement
        /// </summary>
        /// <returns>String representation of the message</returns>
        public override string ToString()
        {
            using (new LockAndElevate(Lock))
                return "Msg<" +
                        sender + "::" + vid + ":" + msgid + (nRaw == 0 ? "" : (", nRaw=" + nRaw)) +
                        ((flags & SENTBYORACLE) == 0 ? "" : " as ORACLE") +
                        ", dest=" + dest +
                        (gaddr == null ? "" : (", gaddr=" + gaddr.ToString())) +
                         (Lid <= 0 ? "" : (", Logging-id=" + Lid)) +
                        ", flags = {" + Msg.pflags(flags) + "}" +
                        ((IsisSystem.Debug & IsisSystem.PPAYLOADS) != 0 ? ", Payload=" + PPayload(payload) : "") +
                    ">";
        }

        // Makes an effort to print the payload of a message
        internal static string PPayload(byte[] thePayload)
        {
            if ((IsisSystem.Debug & IsisSystem.PPAYLOADS) == 0)
                return "";
            object[] obs;
            try { obs = Msg.BArrayToObjects(thePayload); }
            catch { return "byte[" + thePayload.Length + "]"; }
            string s = "  [ ";
            if (obs != null && obs.Length > 0)
                if (Msg.CheckTypes(obs, typeof(Address), typeof(Address), typeof(int), typeof(int), typeof(long), typeof(int), typeof(byte), typeof(byte[])))
                    s += new Msg(thePayload).ToString();
                else if (Msg.CheckTypes(obs, typeof(byte), typeof(byte), typeof(int), typeof(Address), typeof(Address), typeof(Address), typeof(int), typeof(byte[])))
                    // type, code, myCounter, Isis.my_address, dest, gaddr, minStable, originalBuffer
                    s += "<" + Msg.mtypes[(byte)obs[0]] + ": dest " + (Address)obs[4] + ", gaddr " + (Address)obs[5] + ":: " + PPayload((byte[])obs[7]) + " > ";
                else foreach (object o in obs)
                        s += o.ToString() + " ";
            return s + "]";
        }

        internal long key(int flags)
        {
            if ((flags & Group.G_ISLARGE) != 0)
                return (long)msgid;
            return (((long)vid) << 32) + (long)msgid;
        }

        /// <summary>
        /// For implementation of the IComparable API
        /// </summary>
        /// <param name="obj"></param>
        /// <returns></returns>
        public override bool Equals(object obj)
        {
            if (obj == null || !obj.GetType().Equals(typeof(Msg)))
                return false;
            return GetHashCode() == ((Msg)obj).GetHashCode();
        }

        /// <summary>
        /// For implementation of the IComparable API
        /// </summary>
        /// <returns></returns>
        public override int GetHashCode()
        {
            return sender.GetHashCode() + vid * 37 + msgid * 11 + (int)(flags & SENTBYORACLE) * 7;
        }

        /// <summary>
        /// For implementation of the IComparable API
        /// </summary>
        /// <returns></returns>
        public int CompareTo(object obj)
        {
            Msg m = (Msg)obj;
            if (vid != m.vid)
                return vid - m.vid;
            else if (msgid != m.msgid)
                return msgid - m.msgid;
            return sender.GetHashCode() - m.sender.GetHashCode();
        }

        internal const byte ADDRESS = 0xFF;
        internal const byte ADDRESSA = 0xFE;
        internal const byte INT16 = 0xFD;
        internal const byte INT16A = 0xFC;
        internal const byte INT32 = 0xFB;
        internal const byte INT32A = 0xFA;
        internal const byte INT2D = 0xF9;
        internal const byte INT64 = 0xF8;
        internal const byte INT64A = 0xF7;
        internal const byte BYTE = 0xF6;
        internal const byte BYTEA = 0xF5;
        internal const byte BOOL = 0xF4;
        internal const byte BOOLA = 0xF3;
        internal const byte STRING = 0xF2;
        internal const byte STRINGA = 0xF1;
        internal const byte FLOAT = 0xF0;
        internal const byte FLOATA = 0xEF;
        internal const byte FLOAT2D = 0xEE;
        internal const byte DOUBLE = 0xED;
        internal const byte DOUBLEA = 0xEC;
        internal const byte DOUBLE2D = 0xEB;
        internal const byte CHAR = 0xEA;
        internal const byte CHARA = 0xE9;
        internal const byte UINT16 = 0xE8;
        internal const byte UINT16A = 0xE7;
        internal const byte UINT32 = 0xE6;
        internal const byte UINT32A = 0xE5;
        internal const byte UINT64 = 0xE4;
        internal const byte UINT64A = 0xE3;
        internal const byte MESSAGE = 0xE2;
        internal const byte VIEW = 0xE1;
        internal const byte VIEWDELTA = 0xE0;
        internal const byte NESTED = 0xDF;
        internal const byte NESTED0 = 0xDE;
        internal const byte NULL = 0xDD;
        internal const byte UNSTABLE = 0xDC;
        internal const byte TOKENINFO = 0xDB;
        internal const byte GRPAIR = 0xDA;
        internal const byte FLSHAGGKEY = 0xD0;
        internal const byte BVS = 0xD9;
        internal const byte TCPHDR = 0xD8;
        internal const byte LOCKINFO = 0xD7;
        internal const byte KEYVALUEP = 0xD6;
        internal const byte LIST = 0xD5;
        internal const byte QUERYKEY = 0xD4;
        internal const byte DHTITEM = 0xD3;
        internal const byte OSSPQ = 0xD2;
        internal const byte LISTO = 0xD1;
        internal const byte OOBREPINFO = 0xCF;
        internal const byte UNDEF = 0xCE;

        // Larger than SYSTEM reserved
        internal const byte SYSTEM = 0xCF;

        internal static Dictionary<Type, byte> UserDefinedTypesList;
        internal static UDT[] UserDefinedTypesTable = new UDT[0xFF];
        internal static LockObject UDTLock = new LockObject("UDTLock");
        internal static List<Type> AutoMarshalledList = new List<Type>();

        internal static void Initialize()
        {
            if (UserDefinedTypesList != null) return;

            // Initializes the myTypes table and registers "Msg" as a user-defined type
            UserDefinedTypesList = new Dictionary<Type, byte>(256);

            // These types support value, vector and 2-D arrays
            UserDefinedTypesList[typeof(Int32)] = INT32; UserDefinedTypesList[typeof(Int32[])] = INT32A; UserDefinedTypesList[typeof(Int32[,])] = INT2D;
            UserDefinedTypesList[typeof(float)] = FLOAT; UserDefinedTypesList[typeof(float[])] = FLOATA; UserDefinedTypesList[typeof(float[,])] = FLOAT2D;
            UserDefinedTypesList[typeof(double)] = DOUBLE; UserDefinedTypesList[typeof(double[])] = DOUBLEA; UserDefinedTypesList[typeof(double[,])] = DOUBLE2D;

            // These support just values and vectors
            UserDefinedTypesList[typeof(Int16)] = INT16; UserDefinedTypesList[typeof(Int16[])] = INT16A;
            UserDefinedTypesList[typeof(Int64)] = INT64; UserDefinedTypesList[typeof(Int64[])] = INT64A;
            UserDefinedTypesList[typeof(byte)] = BYTE; UserDefinedTypesList[typeof(byte[])] = BYTEA;
            UserDefinedTypesList[typeof(bool)] = BOOL; UserDefinedTypesList[typeof(bool[])] = BOOLA;
            UserDefinedTypesList[typeof(string)] = STRING; UserDefinedTypesList[typeof(string[])] = STRINGA;
            UserDefinedTypesList[typeof(char)] = CHAR; UserDefinedTypesList[typeof(char[])] = CHARA;
            UserDefinedTypesList[typeof(UInt16)] = UINT16; UserDefinedTypesList[typeof(UInt16[])] = UINT16A;
            UserDefinedTypesList[typeof(UInt32)] = UINT32; UserDefinedTypesList[typeof(UInt32[])] = UINT32A;
            UserDefinedTypesList[typeof(UInt64)] = UINT64; UserDefinedTypesList[typeof(UInt64[])] = UINT64A;
            UserDefinedTypesList[typeof(Address)] = ADDRESS; UserDefinedTypesList[typeof(Address[])] = ADDRESSA;
            UserDefinedTypesList[typeof(byte[][])] = BVS;

            // System-defined Message type
            doRegisterType(typeof(Msg), MESSAGE);
            doRegisterType(typeof(View), VIEW);
        }

        static Msg() { Initialize(); }

        internal class UDT
        {
            internal Type theType;
            internal MethodInfo theMarshaller;
            internal ConstructorInfo theConstructor;
            internal FieldInfo[] theFields;
            internal int index;

            internal UDT(Type t, MethodInfo mi, ConstructorInfo ci, FieldInfo[] flds, int idx)
            {
                theType = t;
                theMarshaller = mi;
                theConstructor = ci;
                if (flds != null && flds.Length == 0)
                    throw new IsisException("Automarshaller: class " + t + " has no public instance fields.");
                theFields = flds;
                index = idx;
            }
        }

        /// <summary>
        /// Registers a user-defined class to be marshalled into and out of byte vectors for transmission.
        /// </summary>
        /// <remarks>
        /// A user defined type must be register itself with Isis via a call to Msg.RegisterType prior to the first use.  
        /// There are two subcases.  If the class has the attribute [AutoMarshalled] then Isis will automatically scan the object
        /// instance, creating a vector of objects from the public fields, and then invoking <see cref="Msg.toBArray()"/>.
        /// For incoming data, Isis uses <see cref="Msg.BArrayToObjects(byte[])"/> to decode the byte[] representation into an object
        /// vector, then assigns the objects to the corresponding public fields, in order.  
        /// The second case omits the [AutoMarshalled] attribute.  In this case the user defined type should support the following interface
        ///       public byte[] toBArray():       converts "this" to a byte array, perhaps using Msg.toBArray(list of objects) for help
        ///       public class-name(object[] obs): a constructor that, given an object vector, converts it back to an instance of class-name
        /// </remarks>
        ///       
        /// <seealso cref="Msg.toBArray(object[])"/> and <seealso cref="Msg.BArrayToObjects(byte[])"/>
        /// <param name="T"></param>
        /// <param name="TID"></param>
        public static void RegisterType(Type T, byte TID)
        {
            if (TID >= SYSTEM) throw new ArgumentOutOfRangeException("Msg.RegisterType: invalid TID");
            doRegisterType(T, TID);
        }

        internal static void doRegisterType(Type T, byte TID)
        {
            bool autoMarshall = false;
            using (new LockAndElevate(UDTLock))
            {
                if (UserDefinedTypesTable[TID] != null)
                    if (UserDefinedTypesTable[TID].theType.Equals(T))
                        return;
                    else
                        throw new IsisException("Attempt to re-register Isis.Types[" + TID + "] as type " + T);
                object[] atts = T.GetCustomAttributes(true);
                foreach (object att in atts)
                    if (att.GetType().Equals(typeof(AutoMarshalled)))
                    {
                        AutoMarshalledList.Add(T);
                        autoMarshall = true;
                        break;
                    }
                if (autoMarshall)
                {
                    ConstructorInfo theConstructor = T.GetConstructor(Type.EmptyTypes);
                    if (theConstructor == null) throw new IsisException("Msg.RegisterType: Can't find constructor for auto-marshalled type with signature " + T + "()");
                    UserDefinedTypesTable[TID] = new UDT(T, null, theConstructor, T.GetFields(BindingFlags.Instance | BindingFlags.Public), TID);
                }
                else
                {
                    if (T.GetMethod("toBArray", BindingFlags.Instance | BindingFlags.Public) == null) throw new IsisException("Msg.RegisterType: class " + T + " doesn't define xx.toBArray() method");
                    if (T.GetConstructor(new Type[] { typeof(byte[]) }) == null) throw new IsisException("Msg.RegisterType: Can't find constructor for " + T + "(byte[])");
                    UserDefinedTypesTable[TID] = new UDT(T, T.GetMethod("toBArray", BindingFlags.Instance | BindingFlags.Public), T.GetConstructor(new Type[] { typeof(byte[]) }), null, TID);
                }
                UserDefinedTypesList[T] = TID;
            }
        }

        internal static bool CheckTypes(object[] objs, params Type[] types)
        {
            if (objs.Length != types.Length)
                return false;
            for (int idx = 0; idx < objs.Length; idx++)
                if (!objs[idx].GetType().Equals(types[idx]))
                    return false;
            return true;
        }

        internal static void ConfirmRegistered(Type T)
        {
            if (T.IsArray)
                T = T.GetElementType();
            else if (T.IsGenericType)
                T = T.GetGenericTypeDefinition();
            if (UserDefinedTypesList.ContainsKey(T))
                return;
            throw new IsisException("Aggregator<" + T + ">: type must be serializable via Isis.Msg (did you forget to call Isis.Msg.RegisterType?)");

        }

        internal static bool IsAscii(string s)
        {
            foreach (char c in s)
            {
                int asInt = (int)c;
                if (asInt < 1 || asInt > 127)
                    return false;
            }
            return true;
        }

        internal static bool IsAscii(byte[] bv, int off, int len)
        {
            for (int idx = off; len-- > 0; idx++)
            {
                byte b = bv[idx];
                if (b < 1 || b > 127)
                    return false;
            }
            return true;
        }

        internal static byte[] StringToBytes(string s)
        {
            if (IsAscii(s))
                return ASCIIEncoding.ASCII.GetBytes(s);
            else
                return Encoding.Unicode.GetBytes(s);
        }

        internal static string BytesToString(byte[] b, int off, int len)
        {
            if (IsAscii(b, off, len))
                return ASCIIEncoding.ASCII.GetString(b, off, len);
            else
                return Encoding.Unicode.GetString(b, off, len);
        }

        // Used to preserve strong typing "to the last step"
        internal static byte[] NewMsgAsBArray(Address sender, Address gaddr, int vid, int msgid, long Lid, int nRaw, int UID, byte flag, byte[] buffer)
        {
            return toBArray(sender, gaddr, vid, msgid, Lid, nRaw, UID, flag, buffer);
        }

        /// <summary>
        /// Marshalls a vector of objects into a byte[] vector for storage or transmission
        /// </summary>
        /// <param name="obs">the vector of objects</param>
        /// <returns>a byte[] vector containing a concise encoding of the objects</returns>
        public static byte[] toBArray(params object[] obs)
        {
            return _toBArray(true, true, obs);
        }

        internal static byte[] ReplyToBArray(params object[] obs)
        {
            return _toBArray(true, false, obs);
        }


        private static byte[] _toBArray(bool topLevel, object[] obs)
        {
            return _toBArray(topLevel, true, obs);
        }

        // At top level, signs the byte[] array; for inner-level leaves zeros.
        private static byte[] _toBArray(bool topLevel, bool unpackInner, object[] obs)
        {
            if (unpackInner && obs.Length == 1 && obs[0].GetType().Equals(typeof(object[])))
                obs = (object[])obs[0];
            byte[] payload;
            List<byte[]> contents = new List<byte[]>();
            int payloadLen = 0, index = 0, ctindex = 0;
            byte[] ctypes = new byte[obs.Length];
            foreach (object o in obs)
            {
                if (o == null)
                    throw new IsisException("Isis2 unable to mashall object to byte stream (null reference: lacks type information)");
                Type t = o.GetType();
                bool isArray;
                byte code;
                Type gt = t.IsGenericType ? t.GetGenericTypeDefinition() : null;
                isArray = false;
                code = UNDEF;
                if (o == null)
                    code = NULL;
                else if (gt != null)
                {
                    if (gt.Equals(typeof(KeyValuePair<,>)))
                        code = KEYVALUEP;
                    else if (gt.Equals(typeof(List<>)))
                        code = LIST;
                    else if (gt.Equals(typeof(QueryKey<>)))
                        code = QUERYKEY;
                    else if (t.GetInterfaces().Any(it => it.IsGenericType && it.GetGenericTypeDefinition().Equals(typeof(IEnumerable<>))))
                        Isis.WriteLine("WARNING: Isis2 can't serialize IEnumerable<T>; use List<T> or Array<T> instead");
                }
                else
                    UserDefinedTypesList.TryGetValue(t, out code);
                switch (code)
                {
                    case NULL:
                        {
                            byte[] nb = new byte[4];
                            ctypes[ctindex++] = NULL;
                            contents.Add(nb);
                            payloadLen += 4;
                        }
                        continue;

                    case INT16:
                        {
                            Int16 value = (Int16)o;
                            byte[] ba = new byte[2];
                            index = 0;
                            ba[index++] = (byte)((value >> 8) & 0xFF);
                            ba[index++] = (byte)(value & 0xFF);
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = INT16;
                        }
                        continue;

                    case INT16A:
                        {
                            Int16[] values = (Int16[])o;
                            byte[] ba = new byte[2 * values.Length];
                            index = 0;
                            foreach (Int16 value in values)
                            {
                                ba[index++] = (byte)((value >> 8) & 0xFF);
                                ba[index++] = (byte)(value & 0xFF);
                            }
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = INT16A;
                        }
                        continue;

                    case UINT16:
                        {
                            UInt16 value = (UInt16)o;
                            byte[] ba = new byte[2];
                            index = 0;
                            ba[index++] = (byte)((value >> 8) & 0xFF);
                            ba[index++] = (byte)(value & 0xFF);
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = UINT16;
                        }
                        continue;

                    case UINT16A:
                        {
                            UInt16[] values = (UInt16[])o;
                            byte[] ba = new byte[2 * values.Length];
                            index = 0;
                            foreach (UInt16 value in values)
                            {
                                ba[index++] = (byte)((value >> 8) & 0xFF);
                                ba[index++] = (byte)(value & 0xFF);
                            }
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = UINT16A;
                        }
                        continue;

                    case INT32:
                        {
                            Int32 value = (Int32)o;
                            byte[] ba = new byte[4];
                            index = 0;
                            ba[index++] = (byte)((value >> 24) & 0xFF);
                            ba[index++] = (byte)((value >> 16) & 0xFF);
                            ba[index++] = (byte)((value >> 8) & 0xFF);
                            ba[index++] = (byte)(value & 0xFF);
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = INT32;
                        }
                        continue;

                    case INT32A:
                        {
                            Int32[] values = (Int32[])o;
                            byte[] ba = new byte[4 * values.Length];
                            index = 0;
                            foreach (Int32 value in values)
                            {
                                ba[index++] = (byte)((value >> 24) & 0xFF);
                                ba[index++] = (byte)((value >> 16) & 0xFF);
                                ba[index++] = (byte)((value >> 8) & 0xFF);
                                ba[index++] = (byte)(value & 0xFF);
                            }
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = INT32A;
                        }
                        continue;

                    case UINT32:
                        {
                            UInt32 value = (UInt32)o;
                            byte[] ba = new byte[4];
                            index = 0;
                            ba[index++] = (byte)((value >> 24) & 0xFF);
                            ba[index++] = (byte)((value >> 16) & 0xFF);
                            ba[index++] = (byte)((value >> 8) & 0xFF);
                            ba[index++] = (byte)(value & 0xFF);
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = UINT32;
                        }
                        continue;

                    case UINT32A:
                        {
                            UInt32[] values = (UInt32[])o;
                            byte[] ba = new byte[4 * values.Length];
                            index = 0;
                            foreach (UInt32 value in values)
                            {
                                ba[index++] = (byte)((value >> 24) & 0xFF);
                                ba[index++] = (byte)((value >> 16) & 0xFF);
                                ba[index++] = (byte)((value >> 8) & 0xFF);
                                ba[index++] = (byte)(value & 0xFF);
                            }
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = UINT32A;
                        }
                        continue;

                    case INT2D:
                        {
                            Int32[,] Ivalues = (Int32[,])o;
                            using (MemoryStream memStream = new MemoryStream(100))
                            {
                                BinaryFormatter myBf = new BinaryFormatter();
                                myBf.Serialize(memStream, Ivalues);
                                memStream.Seek(0, SeekOrigin.Begin);
                                byte[] ba = new byte[memStream.Length];
                                memStream.Read(ba, 0, (int)memStream.Length);
                                memStream.SetLength(0);
                                contents.Add(ba);
                                payloadLen += ba.Length;
                            }
                            ctypes[ctindex++] = INT2D;
                        }
                        continue;

                    case INT64:
                        {
                            Int64 value = (Int64)o;
                            byte[] ba = new byte[8];
                            index = 0;
                            ba[index++] = (byte)((value >> 56) & 0xFF);
                            ba[index++] = (byte)((value >> 48) & 0xFF);
                            ba[index++] = (byte)((value >> 40) & 0xFF);
                            ba[index++] = (byte)((value >> 32) & 0xFF);
                            ba[index++] = (byte)((value >> 24) & 0xFF);
                            ba[index++] = (byte)((value >> 16) & 0xFF);
                            ba[index++] = (byte)((value >> 8) & 0xFF);
                            ba[index++] = (byte)(value & 0xFF);
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = INT64;
                        }
                        continue;

                    case INT64A:
                        {
                            Int64[] values = (Int64[])o;
                            byte[] ba = new byte[8 * values.Length];
                            index = 0;
                            foreach (Int64 value in values)
                            {
                                ba[index++] = (byte)((value >> 56) & 0xFF);
                                ba[index++] = (byte)((value >> 48) & 0xFF);
                                ba[index++] = (byte)((value >> 40) & 0xFF);
                                ba[index++] = (byte)((value >> 32) & 0xFF);
                                ba[index++] = (byte)((value >> 24) & 0xFF);
                                ba[index++] = (byte)((value >> 16) & 0xFF);
                                ba[index++] = (byte)((value >> 8) & 0xFF);
                                ba[index++] = (byte)(value & 0xFF);
                            }
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = INT64A;
                        }
                        continue;

                    case UINT64:
                        {
                            UInt64 value = (UInt64)o;
                            byte[] ba = new byte[8];
                            index = 0;
                            ba[index++] = (byte)((value >> 56) & 0xFF);
                            ba[index++] = (byte)((value >> 48) & 0xFF);
                            ba[index++] = (byte)((value >> 40) & 0xFF);
                            ba[index++] = (byte)((value >> 32) & 0xFF);
                            ba[index++] = (byte)((value >> 24) & 0xFF);
                            ba[index++] = (byte)((value >> 16) & 0xFF);
                            ba[index++] = (byte)((value >> 8) & 0xFF);
                            ba[index++] = (byte)(value & 0xFF);
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = UINT64;
                        }
                        continue;

                    case UINT64A:
                        {
                            UInt64[] values = (UInt64[])o;
                            byte[] ba = new byte[8 * values.Length];
                            index = 0;
                            foreach (UInt64 value in values)
                            {
                                ba[index++] = (byte)((value >> 56) & 0xFF);
                                ba[index++] = (byte)((value >> 48) & 0xFF);
                                ba[index++] = (byte)((value >> 40) & 0xFF);
                                ba[index++] = (byte)((value >> 32) & 0xFF);
                                ba[index++] = (byte)((value >> 24) & 0xFF);
                                ba[index++] = (byte)((value >> 16) & 0xFF);
                                ba[index++] = (byte)((value >> 8) & 0xFF);
                                ba[index++] = (byte)(value & 0xFF);
                            }
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = UINT64A;
                        }
                        continue;

                    case BYTE:
                        {
                            byte[] ba = new byte[] { (byte)o };
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = BYTE;
                            continue;
                        }

                    case BYTEA:
                        {
                            contents.Add((byte[])o);
                            payloadLen += ((byte[])o).Length;
                            ctypes[ctindex++] = BYTEA;
                        }
                        continue;

                    case BOOL:
                        {
                            byte[] ba = new byte[] { (byte)(((bool)o) ? (byte)0x1 : (byte)0x0) };
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = BOOL;
                        }
                        continue;

                    case BOOLA:
                        {
                            bool[] value = (bool[])o;
                            byte[] ba = BitsToBytes(value);
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = BOOLA;
                        }
                        continue;

                    case CHAR:
                        {
                            byte[] ba = new byte[] { (byte)((char)o) };
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = CHAR;
                        }
                        continue;

                    case CHARA:
                        {
                            char[] value = (char[])o;
                            byte[] ba = new byte[value.Length + 1];
                            for (int i = 0; i < value.Length; i++)
                                ba[i] = (byte)value[i];
                            ba[value.Length] = (byte)0x00;
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = CHARA;
                        }
                        continue;

                    case STRING:
                        {
                            byte[] ba = StringToBytes((string)o);
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = STRING;
                        }
                        continue;

                    case STRINGA:
                        {
                            int len = 0;
                            index = 0;
                            string[] values = (string[])o;
                            foreach (string value in values)
                                if (value == null)
                                    len += 1;
                                else
                                    len += value.Length + 1;
                            byte[] ba = new byte[len];
                            foreach (string value in values)
                            {
                                if (value != null)
                                    for (int i = 0; i < value.Length; i++)
                                        ba[index++] = (byte)value[i];
                                ba[index++] = (byte)0x00;
                            }
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = STRINGA;
                        }
                        continue;

                    case FLOAT:
                        {
                            float f = (float)o;
                            using (MemoryStream memStream = new MemoryStream(100))
                            {
                                BinaryFormatter myBf = new BinaryFormatter();
                                myBf.Serialize(memStream, f);
                                memStream.Seek(0, SeekOrigin.Begin);
                                byte[] ba = new byte[memStream.Length];
                                memStream.Read(ba, 0, (int)memStream.Length);
                                memStream.SetLength(0);
                                contents.Add(ba);
                                payloadLen += ba.Length;
                            }
                            ctypes[ctindex++] = FLOAT;
                        }
                        continue;

                    case FLOATA:
                        {
                            float[] Fvalues = (float[])o;
                            using (MemoryStream memStream = new MemoryStream(100))
                            {
                                BinaryFormatter myBf = new BinaryFormatter();
                                myBf.Serialize(memStream, Fvalues);
                                memStream.Seek(0, SeekOrigin.Begin);
                                byte[] ba = new byte[memStream.Length];
                                memStream.Read(ba, 0, (int)memStream.Length);
                                memStream.SetLength(0);
                                contents.Add(ba);
                                payloadLen += ba.Length;
                            }
                            ctypes[ctindex++] = FLOATA;
                        }
                        continue;

                    case FLOAT2D:
                        {
                            float[,] Fvalues = (float[,])o;
                            using (MemoryStream memStream = new MemoryStream(100))
                            {
                                BinaryFormatter myBf = new BinaryFormatter();
                                myBf.Serialize(memStream, Fvalues);
                                memStream.Seek(0, SeekOrigin.Begin);
                                byte[] ba = new byte[memStream.Length];
                                memStream.Read(ba, 0, (int)memStream.Length);
                                memStream.SetLength(0);
                                contents.Add(ba);
                                payloadLen += ba.Length;
                            }
                            ctypes[ctindex++] = FLOAT2D;
                        }
                        continue;

                    case DOUBLE:
                        {
                            double f = (double)o;
                            using (MemoryStream memStream = new MemoryStream(100))
                            {
                                BinaryFormatter myBf = new BinaryFormatter();
                                myBf.Serialize(memStream, f);
                                byte[] ba = new byte[memStream.Length];
                                memStream.Seek(0, SeekOrigin.Begin);
                                memStream.Read(ba, 0, (int)memStream.Length);
                                memStream.SetLength(0);
                                contents.Add(ba);
                                payloadLen += ba.Length;
                            }
                            ctypes[ctindex++] = DOUBLE;
                        }
                        continue;

                    case DOUBLEA:
                        {
                            double[] Fvalues = (double[])o;
                            using (MemoryStream memStream = new MemoryStream(100))
                            {
                                BinaryFormatter myBf = new BinaryFormatter();
                                myBf.Serialize(memStream, Fvalues);
                                memStream.Seek(0, SeekOrigin.Begin);
                                byte[] ba = new byte[memStream.Length];
                                memStream.Read(ba, 0, (int)memStream.Length);
                                memStream.SetLength(0);
                                contents.Add(ba);
                                payloadLen += ba.Length;
                            }
                            ctypes[ctindex++] = DOUBLEA;
                        }
                        continue;

                    case DOUBLE2D:
                        {
                            double[,] Dvalues = (double[,])o;
                            using (MemoryStream memStream = new MemoryStream(100))
                            {
                                BinaryFormatter myBf = new BinaryFormatter();
                                myBf.Serialize(memStream, Dvalues);
                                memStream.Seek(0, SeekOrigin.Begin);
                                byte[] ba = new byte[memStream.Length];
                                memStream.Read(ba, 0, (int)memStream.Length);
                                memStream.SetLength(0);
                                contents.Add(ba);
                                payloadLen += ba.Length;
                            }
                            ctypes[ctindex++] = DOUBLE2D;
                        }
                        continue;

                    case ADDRESS:
                        {
                            byte[] ba = ((Address)o).toBArray();
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = ADDRESS;
                        }
                        continue;

                    case ADDRESSA:
                        {
                            byte[] ba = Address.toBArray(((Address[])o));
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = ADDRESSA;
                        }
                        continue;

                    case BVS:
                        {
                            int[] lens = new int[((byte[][])o).Length];
                            int idx = 0, totalBytes = 0;
                            foreach (byte[] bv in (byte[][])o)
                                totalBytes += lens[idx++] = bv.Length;
                            byte[] bvs = new byte[totalBytes];
                            idx = 0;
                            foreach (byte[] bv in (byte[][])o)
                            {
                                Array.Copy(bv, 0, bvs, idx, bv.Length);
                                idx += bv.Length;
                            }
                            byte[] ba = _toBArray(false, new object[] { lens, bvs });
                            contents.Add(ba);
                            payloadLen += ba.Length;
                            ctypes[ctindex++] = BVS;
                        }
                        continue;

                    case KEYVALUEP:
                        {
                            object key = o.GetType().GetMethod("get_Key").Invoke(o, new object[0]);
                            object value = o.GetType().GetMethod("get_Value").Invoke(o, new object[0]);
                            byte[] nb = _toBArray(false, false, new object[] { key, value });
                            ctypes[ctindex++] = KEYVALUEP;
                            contents.Add(nb);
                            payloadLen += nb.Length;
                        }
                        continue;

                    case QUERYKEY:
                        {
                            byte[] nb = ((QKD)o).ToBArray();
                            ctypes[ctindex++] = QUERYKEY;
                            contents.Add(nb);
                            payloadLen += nb.Length;
                        }
                        continue;

                    case LIST:
                        {
                            List<byte[]> tmp = new List<byte[]>();
                            Type it = o.GetType().GetGenericArguments()[0];
                            byte[] nb;
                            bool empty = true;
                            foreach (var x in (IEnumerable)o)
                            {
                                tmp.Add(_toBArray(false, false, new object[] { x }));
                                empty = false;
                            }
                            if (!empty || it.Equals(typeof(object)))
                                nb = _toBArray(false, false, new object[] { tmp.ToArray() });
                            else
                            {
                                code = UNDEF;
                                UserDefinedTypesList.TryGetValue(it, out code);
                                if (code == UNDEF || code == 0)
                                    throw new IsisException("Isis is unable to marshall zero-length lists of type " + it + " @ " + Isis.ExtractStackTrace());
                                nb = new byte[] { code };
                            }
                            ctypes[ctindex++] = it.Equals(typeof(object)) ? LISTO : LIST;
                            contents.Add(nb);
                            payloadLen += nb.Length;
                        }
                        continue;

                    default:
                        {
                            bool fnd = false;
                            if (t.IsArray)
                            {
                                isArray = true;
                                do
                                    t = t.GetElementType();
                                while (t.IsArray);
                            }
                            UDT udt = null;
                            byte which = 0;
                            bool fndT = false;
                            if (UserDefinedTypesList.TryGetValue(t, out which))
                            {
                                fndT = true;
                                udt = UserDefinedTypesTable[which];
                            }
                            if (fndT)
                            {
                                fnd = true;
                                // A type registered with Isis
                                if (isArray == false && udt != null)
                                {
                                    MethodInfo mi;
                                    mi = udt.theMarshaller;
                                    object[] args = new object[0];
                                    if ((IsisSystem.Debug & IsisSystem.CALLBACKS) != 0)
                                        Group.ReportCb(t, o);
                                    byte[] ba;
                                    if (mi != null)
                                        ba = (byte[])mi.Invoke(o, args);
                                    else
                                    {
                                        // AutoMarshalled: Generates a byte[] from the full set of public fields of the object
                                        FieldInfo[] theFields = UserDefinedTypesTable[which].theFields;
                                        Object[] objects = new Object[theFields.Length];
                                        int idx = 0;
                                        foreach (FieldInfo fi in theFields)
                                            objects[idx++] = fi.GetValue(o);
                                        ba = _toBArray(false, objects);
                                    }
                                    contents.Add(ba);
                                    payloadLen += ba.Length;
                                    ctypes[ctindex++] = which;
                                    break;
                                }
                                else // An array (easy) or an array of arrays (requires recursion)
                                {
                                    byte[] ba;
                                    if (((object[])o).Length > 0)
                                    {
                                        object[] objs = new object[((object[])o).Length];
                                        for (int i = 0; i < objs.Length; i++)
                                            objs[i] = ((object[])o)[i];
                                        ba = _toBArray(false, objs);
                                        contents.Add(ba);
                                        payloadLen += ba.Length;
                                        ctypes[ctindex++] = NESTED;
                                        break;
                                    }
                                    // Special case: a 0-length vector of a user-defined object type
                                    ctypes[ctindex++] = NESTED0;
                                    ba = new byte[] { (byte)which };
                                    contents.Add(ba);
                                    payloadLen += 1;
                                    break;
                                }
                            }
                            if (!fnd)
                                throw new IsisException("Msg.toBArray is unable to marshall object of type " + o.GetType() + " (did you remember to call Msg.RegisterType?)");
                        }
                        continue;
                }
            }
            index = ctypes.Length * 5 + 4;
            // The byte array is padded with a single-byte null, then length rounded up to a multiple of 8
            // the last byte will be incremented by Resend() each time it resends a serialized message, just to change the message
            // slightly
            int pln = 4 + index + payloadLen + Isis.ISIS_MSGPADDING;
            pln = (pln + 7) & ~7;
            payload = new byte[pln];
            int off = 0, coff = 0;
            payload[off] = (byte)((ctypes.Length >> 24) & 0xFF); ++off;
            payload[off] = (byte)((ctypes.Length >> 16) & 0xFF); ++off;
            payload[off] = (byte)((ctypes.Length >> 8) & 0xFF); ++off;
            payload[off] = (byte)(ctypes.Length & 0xFF); ++off;
            foreach (byte[] pbs in contents)
            {
                int len = pbs.Length;
                payload[off] = (byte)ctypes[coff]; ++off; ++coff;
                payload[off] = (byte)((len >> 24) & 0xFF); ++off;
                payload[off] = (byte)((len >> 16) & 0xFF); ++off;
                payload[off] = (byte)((len >> 8) & 0xFF); ++off;
                payload[off] = (byte)(len & 0xFF); ++off;
                for (int i = 0; i < pbs.Length; i++)
                    payload[index++] = pbs[i];
            }
            if (Isis.ISIS_MD5SIGS && topLevel)
            {
                using (MemoryStream ms = new MemoryStream(payload, 0, pln - Isis.ISIS_MSGPADDING))
                {
                    using (HMAC hm = new HMACMD5(new byte[8] { 56, 78, 9, 23, 10, 87, 33, 11 }))
                    {
                        byte[] hash = hm.ComputeHash(ms);
                        if (Isis.ISIS_AES != null)
                            hash = sign(hash);
                        if (hash.Length != Isis.ISIS_MSGPADDING)
                            throw new IsisException("signature len");
                        index = payload.Length - hash.Length;
                        for (int i = 0; i < hash.Length; i++)
                            payload[index++] = hash[i];
                    }
                }
            }
            return payload;
        }

        /// <summary>
        /// InvokeFromBArrays first demarshalls the given byte vectors to create a single new vector,
        /// each entry in which is a vector of objects (<seealso cref="MsgToObjects(Msg, Type[])"/>).  It then sorts and merges these to produce one vector per object type,
        /// and finally invokes the method specified by the delegate argument, after checking that the types of the object vectors match the types of the parameters to the method.
        /// 
        /// For example, if you have 4 messages, each containing an int and a double, your delegate will be involked with two arguments, the first being a vector of four ints,
        /// and the second a vector of 4 doubles. Throws an IsisException if any mismatch is detected. 
        /// </summary>
        /// <param name="barrays">byte[] vectors from which to extract objects</param>
        /// <param name="del">delegate to a method that has vector parameters matching those of the expected objects</param>
        public static void InvokeFromBArrays(List<byte[]> barrays, Delegate del)
        {
            if ((IsisSystem.Debug & IsisSystem.MESSAGELAYER) != 0)
                Isis.WriteLine("InvokeFromBArrays(byte[" + barrays.Count() + "][] delegate " + del);
            Type[] types = GetTypes(del);
            doInvokeArray(del, barrays, types);
        }

        internal static void BArrayToVals(object[] resRefs, byte[] barray)
        {
            object[] obs = Msg.BArrayToObjects(barray);
            if (obs.Length != resRefs.Length)
                throw new IsisException("Isis.BArrayToVals: response has different number of objects than expected");
            for (int j = 0; j < obs.Length; j++)
                if (obs[j] != null && obs[j].GetType().Equals(resRefs[j].GetType()) == false)
                    throw new IsisException("Isis.BArrayToVals: " + j + "'th object is of type type " + obs[j].GetType() + " but expected " + resRefs[j].GetType());
                else if (obs[j] != null)
                    ((Array)resRefs[j]).SetValue(obs[j], 0);
        }

        internal static void BArraysToLists(object[] resRefs, List<byte[]> barrays)
        {
            for (int i = 0; i < barrays.Count(); i++)
            {
                object[] obs = Msg.BArrayToObjects(barrays[i]);
                // This garish code snippet compensates for a C# "feature" (aka bug) that causes g.Reply(myClass[]) to be treated like g.Reply(myClass0, myClass1, ...) 
                Type gat = null;
                if (resRefs.Length == 1)
                {
                    Type[] ga = resRefs[0].GetType().GetGenericArguments();
                    if (ga != null && ga.Length == 1)
                        gat = ga[0];
                }
                if (obs.Length != resRefs.Length || (obs.Length > 0 && gat != null && !obs[0].GetType().IsArray && gat.IsArray))
                {
                    Type t;
                    if (obs.Length > 0 && resRefs.Length == 1 && AllHaveSameType(obs, out t))
                    {
                        // This is the case C# (deliberately) misreports to Isis; change the thing back into a vector.
                        object[] newobs = (object[])System.Array.CreateInstance(t, obs.Length);
                        int idx = 0;
                        foreach (object o in obs)
                            newobs[idx++] = o;
                        obs = new object[] { newobs };
                    }
                    else
                    {
                        Isis.WriteLine("WARNING: Isis.BArraysToLists: response has different number of objects (" + obs.Length + ") than expected (" + resRefs.Length + ")");
                        return;
                    }
                }
                for (int j = 0; j < obs.Length; j++)
                    if (obs[j] != null)
                        ((IList)resRefs[j]).Add(obs[j]);
            }
        }

        internal static bool AllHaveSameType(object[] obs, out Type t)
        {
            t = null;
            bool allHaveSameType = true;
            foreach (object o in obs)
                if (t == null)
                    t = o.GetType();
                else if (!(allHaveSameType = t.Equals(o.GetType())))
                    break;
            return allHaveSameType;
        }

        /// <summary>
        /// InvokeFromBArray first demarshalls the given byte vector into a vector of objects (<seealso cref="MsgToObjects(Msg, Type[])"/>) and then 
        /// invokes the method specified by the delegate argument, after checking that the types of the objects match the types of the parameters to the method.
        /// Throws an IsisException if any mismatch is detected
        /// </summary>
        /// <param name="barray"></param>
        /// <param name="del"></param>
        /// <returns>True on success, false for mismatched types</returns>
        public static bool InvokeFromBArray(byte[] barray, Delegate del)
        {
            Type[] types = GetTypes(del);
            return doInvokeSingle(del, barray, types);
        }

        /// <summary>
        /// When involked for Isis.Msg m, m.InvokeFromMsgs(Msg[] msgs, delegate) demarshalls the payloads of the vector of messages msgs to create
        /// a vector, each entry in which is a vector of objects (<seealso cref="MsgToObjects(Msg, Type[])"/>).  It then generates one vector per object type,
        /// and finally invokes the method specified by the delegate argument, after checking that the types of the object vectors match the types of the parameters to the method.
        /// 
        /// For example, if you have 4 messages, each containing an int and a double, your delegate will be involked with two arguments, the first being a vector of four ints,
        /// and the second a vector of 4 doubles. Throws an IsisException if any mismatch is detected. 
        /// </summary>
        /// <param name="msgs"></param>
        /// <param name="del"></param>
        public static void InvokeFromMsgs(Msg[] msgs, Delegate del)
        {
            Type[] types = GetTypes(del);
            List<byte[]> barrays = new List<byte[]>();
            for (int i = 0; i < msgs.Length; i++)
                barrays[i] = msgs[i].payload;
            doInvokeArray(del, barrays, types);
        }

        /// <summary>
        /// When involked for Isis.Msg m, m.InvokeFromMsg(delegate) demarshalls the payload of m into a vector of objects (<seealso cref="MsgToObjects(Msg, Type[])"/>) and then 
        /// invokes the method specified by the delegate argument, after checking that the types of the objects match the types of the parameters to the method.
        /// Throws an IsisException if any mismatch is detected
        /// </summary>
        /// <param name="del"></param>
        public void InvokeFromMsg(Delegate del)
        {
            Type[] types = GetTypes(del);
            doInvokeSingle(del, payload, types);
        }

        internal static Type[] GetTypes(Delegate d)
        {
            ParameterInfo[] pi = d.Method.GetParameters();
            Type[] types = new Type[pi.Length];
            for (int i = 0; i < pi.Length; i++)
                types[i] = pi[i].ParameterType;
            return types;
        }

        internal static bool doInvokeSingle(Delegate del, byte[] barray, Type[] types)
        {
            object[] theObs = new object[types.Length];
            object[] obs = BArrayToObjects(true, barray, types);
            Callable cb = new Callable(del);

            if (theObs.Length != obs.Length)
                return false;

            for (int j = 0; j < obs.Length; j++)
                if (obs[j] != null && obs[j].GetType().Equals(types[j]) == false)
                    return false;

            if ((IsisSystem.Debug & IsisSystem.CALLBACKS) != 0)
                Group.ReportCb(cb, obs);
            cb.doUpcall(obs);
            return true;
        }

        internal static void doInvokeArray(Delegate del, List<byte[]> barrays, Type[] types)
        {
            Callable cb = new Callable(del);
            object[] theObs = new object[types.Length];
            for (int i = 0; i < types.Length; i++)
                if (types[i].IsArray)
                    theObs[i] = Array.CreateInstance(types[i].GetElementType(), barrays.Count());
                else
                    throw new IsisException(types[i] + " is a unit (array required)");
            for (int i = 0; i < barrays.Count(); i++)
            {
                object[] obs = BArrayToObjects(barrays[i]);
                for (int j = 0; j < obs.Length; j++)
                    if (obs[j] != null && obs[j].GetType().Equals(types[j].GetElementType()) == false)
                        throw new IsisException("Isis.doInvokeArray: " + j + "'th object in BArray " + i + " is of type type " + obs[j].GetType() + " but expected " + types[j]);
                    else if (obs[j] != null)
                        ((Array)theObs[j]).SetValue(obs[j], i);
            }

            if ((IsisSystem.Debug & IsisSystem.CALLBACKS) != 0)
                Group.ReportCb(cb, theObs);
            cb.doUpcall(theObs);
        }

        /// <summary>
        /// Given an Isis Msg object, converts its payload to a vector of objects, checking that they have the given types,
        /// and returns that vector.  <seealso cref="Msg.BArrayToObjects(byte[], Type[])"/>
        /// </summary>
        /// <param name="m">Message from which to extract objects</param>
        /// <param name="types">Vector of types expected</param>
        /// <returns></returns>
        public static object[] MsgToObjects(Msg m, params Type[] types)
        {
            object[] obs;
            using (new LockAndElevate(m.Lock))
            {
                if (m.myObs != null)
                    obs = m.myObs;
                else
                    obs = BArrayToObjects(m.payload);
                if (obs.Length != types.Length)
                    throw new IsisException("Msg.BArrayToObjects: BArray encodes wrong number of objects (found " + obs.Length + ", expected " + types.Length + ")");
                for (int i = 0; i < obs.Length; i++)
                    if (obs[i] != null && obs[i].GetType() != types[i])
                        throw new IsisException("Msg.BArrayToObjects: " + i + "'th object is of type type " + obs[i].GetType() + " but expected " + types[i]);
                m.myObs = obs;
                return obs;
            }
        }

        /// <summary>
        /// Given an Isis Msg object, converts its payload to a vector of objects and returns that vector.  <seealso cref="Msg.BArrayToObjects(byte[])"/>
        /// </summary>
        /// <param name="m">Message from which to extract objects</param>
        /// <returns></returns>
        public static object[] MsgToObjects(Msg m)
        {
            using (new LockAndElevate(m.Lock))
            {
                if (m.myObs != null)
                    return m.myObs;
                return m.myObs = BArrayToObjects(m.payload);
            }
        }

        /// <summary>
        /// Given a byte[] vector and an array of types, returns an object[] array by demarshalling the byte vector and checks
        /// to confirm that the types match properly.  Throws an IsisException if the number of objects encoded by the byte vector
        /// is incorrect, or if the types don't have an exact, one-to-one match, with the requested types.  This last point
        /// can be a bit of a surprise: subtyping relationships aren't supported in this interface, so the types must be exact
        /// matches or are considered to be unmatched.  For example, if the class Dog extends class Animal, and the byte[] encodes
        /// a Dog object, you must extract it as a Dog object, not as an Animal object, or a general Object.
        /// 
        /// An overload is available for demarshalling vectors containing object types that can't be predicted before the demarshalling
        /// is performed.  However, even those types do need to be known to Isis, either as built-in types Isis supports, or by registering them.
        /// <seealso cref="Msg.RegisterType"/>
        /// </summary>
        /// <param name="payload">byte[] vector from which to extract object</param>
        /// <param name="types">Types expected</param>
        /// <returns></returns>
        public static object[] BArrayToObjects(byte[] payload, params Type[] types)
        {
            return BArrayToObjects(false, payload, types);
        }

        internal static object[] BArrayToObjects(bool inhibitException, byte[] payload, params Type[] types)
        {
            object[] obs = BArrayToObjects(payload);
            if (obs.Length != types.Length || (types.Length == 1 && types[0].IsArray && obs.Length == 1 && !obs[0].GetType().IsArray))
            {
                if (types.Length == 1 && types[0].IsArray)
                {
                    bool allSame = true;
                    foreach (object o in obs)
                        if (!o.GetType().Equals(types[0].GetElementType()))
                        {
                            allSame = false;
                            break;
                        }
                    if (allSame)
                    {
                        object o = Array.CreateInstance(types[0].GetElementType(), obs.Length);
                        for (int i = 0; i < obs.Length; i++)
                            ((Array)o).SetValue(obs[i], i);
                        return new object[] { o };
                    }
                }
                if (inhibitException)
                    return new object[0];
                string ts = "", tts = "";
                foreach (object o in obs)
                    ts += " " + o.GetType() + " ";
                foreach (Type t in types)
                    tts += " " + t + " ";
                throw new IsisException("Msg.BArrayToObjects: BArray encodes wrong number of objects (found " + obs.Length + " { " + ts + " }, expected " + types.Length + "{ " + tts + " })");
            }
            for (int i = 0; i < obs.Length; i++)
                if (obs[i] != null && obs[i].GetType() != types[i])
                    if (inhibitException)
                        return new object[0];
                    else
                        throw new IsisException("Msg.BArrayToObjects: " + i + "'th object is of type type " + obs[i].GetType() + " but expected " + types[i]);
            return obs;
        }

        /// <summary>
        /// Given a byte[] vector, computes an object[] vector by demarshalling the contents.  The types must all be known to Isis, either
        /// as built-in types Isis supports, or by registering them.  <seealso cref="Msg.RegisterType"/>
        /// </summary>
        /// <param name="payload">byte[] vector from which to extract objects</param>
        /// <returns></returns>
        public static object[] BArrayToObjects(byte[] payload)
        {
            if (payload == null)
                return new object[0];
            return NestedBArrayToObjects(payload, 0, payload.Length);
        }

        private static LockObject cacheLock = new LockObject(false, "cacheLock");
        private static byte[] lastPayloadArg;
        private static int lastPayloadLen;                 // Takes advantage of the fact that a ciphered object is ALWAYS larger than a deciphered one
        private static int lastStart;
        private static int lastLen;
        private static object[] lastObs;

        internal static Type doGetType(byte[] payload, int start, int len)
        {
            int index = start + 1;
            byte conv = payload[index++];
            if (UserDefinedTypesTable[conv] != null)
                return UserDefinedTypesTable[conv].theType;
            return null;
        }

        // If payload apparently encodes a single message object, return it.  Else return null.
        internal static Msg InnerMsg(byte[] payload)
        {
            byte[] expected = { 0, 0, 0, 1, MESSAGE, 0, 0, 0, INT64, 0, 0, 0, 9, 
                                  ADDRESS, 0, 0, 0, 16, 
                                  ADDRESS, 0, 0, 0, 16, 
                                  INT32, 0, 0, 0, 4, 
                                  INT32, 0, 0, 0, 4, 
                                  INT64, 0, 0, 0, 8, 
                                  INT32, 0, 0, 0, 4, 
                                  INT32, 0, 0, 0, 4, 
                                  BYTE, 0, 0, 0, 1, 
                                  BYTEA };
            if (payload.Length < 119)
                return null;
            for (int i = 0; i < expected.Length; i++)
                if (payload[i] != expected[i])
                    return null;
            try
            {
                return (Msg)Msg.BArrayToObjects(payload, typeof(Msg))[0];
            }
            catch
            {
                return null;
            }
        }

        private static object[] NestedBArrayToObjects(byte[] payload, int start, int len)
        {
            if (payload.Length < start + len)
                return new object[0];
            using (new LockAndElevate(cacheLock))
                if (lastPayloadArg == payload && lastPayloadLen == payload.Length && lastStart == start && lastLen == len)
                    return lastObs;
            int idx;
            if (payload == null || len == 0)
                return new object[0];
            int index = start;
            int nobs = (int)payload[index++];
            nobs = (nobs << 8) | payload[index++];
            nobs = (nobs << 8) | payload[index++];
            nobs = (nobs << 8) | payload[index++];
            object[] obs = new object[nobs];
            int off = start + 4 + nobs * 5;
            for (int n = 0; n < nobs; n++)
            {
                byte conv = payload[index++];
                int plen = (int)payload[index++];
                plen = (plen << 8) | payload[index++];
                plen = (plen << 8) | payload[index++];
                plen = (plen << 8) | payload[index++];
                switch (conv)
                {
                    case NULL:
                        off += 4;
                        obs[n] = null;
                        continue;

                    case INT16:
                        int thei16Value = (int)payload[off++];
                        thei16Value = (thei16Value << 8) + (int)payload[off++];
                        Int16 i16value = (Int16)thei16Value;
                        obs[n] = (object)i16value;
                        break;

                    case INT16A:
                        Int16[] i16avalue = new Int16[plen / 2];
                        thei16Value = 0;
                        idx = 0;
                        for (int k = 0; k < plen; k += 2)
                        {
                            thei16Value = (int)payload[off++];
                            thei16Value = (thei16Value << 8) + (int)payload[off++];
                            i16avalue[idx++] = (Int16)thei16Value;
                        }
                        obs[n] = (object)i16avalue;
                        break;

                    case UINT16:
                        int theui16Value = (int)payload[off++];
                        theui16Value = (theui16Value << 8) + (int)payload[off++];
                        UInt16 ui16value = (UInt16)theui16Value;
                        obs[n] = (object)ui16value;
                        break;

                    case UINT16A:
                        UInt16[] ui16avalue = new UInt16[plen / 2];
                        theui16Value = 0;
                        idx = 0;
                        for (int k = 0; k < plen; k += 2)
                        {
                            theui16Value = (int)payload[off++];
                            theui16Value = (theui16Value << 8) + (int)payload[off++];
                            ui16avalue[idx++] = (UInt16)theui16Value;
                        }
                        obs[n] = (object)ui16avalue;
                        break;

                    case INT32:
                        int thei32Value = ((int)payload[off] << 24) + ((int)payload[off + 1] << 16) + ((int)payload[off + 2] << 8) + ((int)payload[off + 3]); off += 4;
                        Int32 i32value = (Int32)thei32Value;
                        obs[n] = (object)i32value;
                        break;

                    case INT32A:
                        Int32[] i32avalue = new Int32[plen / 4];
                        idx = 0;
                        for (int k = 0; k < plen; k += 4)
                        {
                            thei32Value = ((int)payload[off] << 24) + ((int)payload[off + 1] << 16) + ((int)payload[off + 2] << 8) + ((int)payload[off + 3]); off += 4;
                            i32value = (Int32)thei32Value;
                            i32avalue[idx++] = i32value;
                        }
                        obs[n] = (object)i32avalue;
                        break;

                    case UINT32:
                        UInt32 theui32Value = ((uint)payload[off] << 24) + ((uint)payload[off + 1] << 16) + ((uint)payload[off + 2] << 8) + ((uint)payload[off + 3]); off += 4;
                        obs[n] = (object)theui32Value;
                        break;

                    case UINT32A:
                        UInt32[] ui32avalue = new UInt32[plen / 4];
                        idx = 0;
                        for (int k = 0; k < plen; k += 4)
                        {
                            theui32Value = ((uint)payload[off] << 24) + ((uint)payload[off + 1] << 16) + ((uint)payload[off + 2] << 8) + ((uint)payload[off + 3]); off += 4;
                            ui32avalue[idx++] = theui32Value;
                        }
                        obs[n] = (object)ui32avalue;
                        break;

                    case INT2D:
                        using (MemoryStream memStream = new MemoryStream(100))
                        {
                            BinaryFormatter myBf = new BinaryFormatter();
                            memStream.Write(payload, off, plen);
                            memStream.Seek(0, SeekOrigin.Begin);
                            off += plen;
                            Int32[,] ia = (Int32[,])myBf.Deserialize(memStream);
                            memStream.SetLength(0);
                            obs[n] = (object)ia;
                        }
                        break;

                    case INT64:
                        Int64 thei64Value = ((Int64)payload[off] << 56) + ((Int64)payload[off + 1] << 48) + ((Int64)payload[off + 2] << 40) + ((Int64)payload[off + 3] << 32); off += 4;
                        thei64Value += ((Int64)payload[off] << 24) + ((Int64)payload[off + 1] << 16) + ((Int64)payload[off + 2] << 8) + ((Int64)payload[off + 3]); off += 4;
                        obs[n] = (object)thei64Value;
                        break;

                    case INT64A:
                        Int64[] i64avalue = new Int64[plen / 8];
                        idx = 0;
                        for (int k = 0; k < plen; k += 8)
                        {
                            thei64Value = ((Int64)payload[off] << 56) + ((Int64)payload[off + 1] << 48) + ((Int64)payload[off + 2] << 40) + ((Int64)payload[off + 3] << 32); off += 4;
                            thei64Value += ((Int64)payload[off] << 24) + ((Int64)payload[off + 1] << 16) + ((Int64)payload[off + 2] << 8) + ((Int64)payload[off + 3]); off += 4;
                            i64avalue[idx++] = thei64Value;
                        }
                        obs[n] = (object)i64avalue;
                        break;

                    case UINT64:
                        ulong theui64Value;
                        theui64Value = ((ulong)payload[off] << 56) + ((ulong)payload[off + 1] << 48) + ((ulong)payload[off + 2] << 40) + ((ulong)payload[off + 3] << 32); off += 4;
                        theui64Value += ((ulong)payload[off] << 24) + ((ulong)payload[off + 1] << 16) + ((ulong)payload[off + 2] << 8) + ((ulong)payload[off + 3]); off += 4;
                        UInt64 ui64value = (UInt64)theui64Value;
                        obs[n] = (object)ui64value;
                        break;

                    case UINT64A:
                        UInt64[] ui64avalue = new UInt64[plen / 8];
                        idx = 0;
                        for (int k = 0; k < plen; k += 8)
                        {
                            theui64Value = ((ulong)payload[off] << 56) + ((ulong)payload[off + 1] << 48) + ((ulong)payload[off + 2] << 40) + ((ulong)payload[off + 3] << 32); off += 4;
                            theui64Value += ((ulong)payload[off] << 24) + ((ulong)payload[off + 1] << 16) + ((ulong)payload[off + 2] << 8) + ((ulong)payload[off + 3]); off += 4;
                            ui64avalue[idx++] = theui64Value;
                        }
                        obs[n] = (object)ui64avalue;
                        break;

                    case BYTE:
                        byte thebValue = payload[off++];
                        obs[n] = (object)thebValue;
                        break;

                    case BYTEA:
                        byte[] bavalue = new byte[plen];
                        idx = 0;
                        for (int k = 0; k < plen; k++)
                            bavalue[idx++] = payload[off++];
                        obs[n] = (object)bavalue;
                        break;

                    case CHAR:
                        char thecValue = (char)payload[off++];
                        obs[n] = (object)thecValue;
                        break;

                    case CHARA:
                        char[] cavalue = new char[plen];
                        idx = 0;
                        for (int k = 0; k < plen; k++)
                            cavalue[idx++] = (char)payload[off++];
                        obs[n] = (object)cavalue;
                        break;

                    case BOOL:
                        bool theboValue = (payload[off++] == 1);
                        obs[n] = (object)theboValue;
                        break;

                    case BOOLA:
                        bavalue = new byte[plen];
                        idx = 0;
                        for (int k = 0; k < plen; k++)
                            bavalue[idx++] = payload[off++];
                        obs[n] = (object)BytesToBits(bavalue);
                        break;

                    case STRING:
                        string thestValue = BytesToString(payload, off, plen);
                        off += plen;
                        obs[n] = (object)thestValue;
                        break;

                    case STRINGA:
                        int ns = 0;
                        for (int k = 0; k < plen; k++)
                            if (payload[off + k] == 0)
                                ++ns;
                        string[] thestavalue = new string[ns];
                        if (ns > 0)
                        {
                            idx = 0;
                            thestavalue[0] = "";
                            for (int k = 0; k < plen; k++)
                                if (payload[off] == 0)
                                {
                                    ++off;
                                    if (++idx != ns)
                                        thestavalue[idx] = "";
                                }
                                else
                                    thestavalue[idx] += (char)payload[off++];
                        }
                        obs[n] = (object)thestavalue;
                        break;

                    case FLOAT:
                        using (MemoryStream memStream = new MemoryStream(100))
                        {
                            BinaryFormatter myBf = new BinaryFormatter();
                            memStream.Write(payload, off, plen);
                            memStream.Seek(0, SeekOrigin.Begin);
                            off += plen;
                            float f = (float)myBf.Deserialize(memStream);
                            memStream.SetLength(0);
                            obs[n] = (object)f;
                        }
                        break;

                    case FLOATA:
                        using (MemoryStream memStream = new MemoryStream(100))
                        {
                            BinaryFormatter myBf = new BinaryFormatter();
                            memStream.Write(payload, off, plen);
                            memStream.Seek(0, SeekOrigin.Begin);
                            off += plen;
                            float[] flts = (float[])myBf.Deserialize(memStream);
                            memStream.SetLength(0);
                            obs[n] = (object)flts;
                        }
                        break;

                    case FLOAT2D:
                        using (MemoryStream memStream = new MemoryStream(100))
                        {
                            BinaryFormatter myBf = new BinaryFormatter();
                            memStream.Write(payload, off, plen);
                            memStream.Seek(0, SeekOrigin.Begin);
                            off += plen;
                            float[,] fa = (float[,])myBf.Deserialize(memStream);
                            memStream.SetLength(0);
                            obs[n] = (object)fa;
                        }
                        break;

                    case DOUBLE:
                        using (MemoryStream memStream = new MemoryStream(100))
                        {
                            BinaryFormatter myBf = new BinaryFormatter();
                            memStream.Write(payload, off, plen);
                            memStream.Seek(0, SeekOrigin.Begin);
                            off += plen;
                            double d = (double)myBf.Deserialize(memStream);
                            memStream.SetLength(0);
                            obs[n] = (object)d;
                        }
                        break;

                    case DOUBLEA:
                        using (MemoryStream memStream = new MemoryStream(100))
                        {
                            BinaryFormatter myBf = new BinaryFormatter();
                            memStream.Write(payload, off, plen);
                            memStream.Seek(0, SeekOrigin.Begin);
                            off += plen;
                            double[] dbls = (double[])myBf.Deserialize(memStream);
                            memStream.SetLength(0);
                            obs[n] = (object)dbls;
                        }
                        break;

                    case DOUBLE2D:
                        using (MemoryStream memStream = new MemoryStream(100))
                        {
                            BinaryFormatter myBf = new BinaryFormatter();
                            memStream.Write(payload, off, plen);
                            memStream.Seek(0, SeekOrigin.Begin);
                            off += plen;
                            double[,] da = (double[,])myBf.Deserialize(memStream);
                            memStream.SetLength(0);
                            obs[n] = (object)da;
                        }
                        break;

                    case ADDRESS:
                        Address a = new Address(payload, off);
                        off += plen;
                        obs[n] = (object)a;
                        break;

                    case ADDRESSA:
                        Address[] av = Address.ToAddresses(payload, off, plen);
                        off += plen;
                        obs[n] = (object)av;
                        break;

                    case BVS:
                        {
                            byte[] barray = new byte[plen];
                            Array.Copy(payload, off, barray, 0, plen);
                            object[] objs = Msg.BArrayToObjects(barray, typeof(int[]), typeof(byte[]));
                            int[] lens = (int[])objs[0];
                            byte[] bvs = (byte[])objs[1];
                            byte[][] res = new byte[lens.Length][];
                            int offset = 0;
                            for (int i = 0; i < lens.Length; i++)
                            {
                                res[i] = new byte[lens[i]];
                                Array.Copy(bvs, offset, res[i], 0, lens[i]);
                                offset += lens[i];
                            }
                            obs[n] = (object)res;
                            off += plen;
                        }
                        break;

                    case NESTED:
                        {
                            object[] nested = Msg.NestedBArrayToObjects(payload, off, plen);
                            if (nested.Length > 0)
                            {
                                obs[n] = System.Array.CreateInstance(nested[0].GetType(), nested.Length);
                                for (int k = 0; k < nested.Length; k++)
                                    ((object[])obs[n])[k] = nested[k];
                            }
                            else
                                obs[n] = System.Array.CreateInstance(Msg.doGetType(payload, off, plen), 0);
                            off += plen;
                        }
                        break;

                    case NESTED0:
                        byte userType = payload[off++];
                        obs[n] = Array.CreateInstance((Type)UserDefinedTypesTable[userType].theType, 0);
                        break;

                    case KEYVALUEP:
                        {
                            object[] nested = Msg.NestedBArrayToObjects(payload, off, plen);
                            obs[n] = Activator.CreateInstance(typeof(KeyValuePair<,>).MakeGenericType(new Type[] { nested[0].GetType(), nested[1].GetType() }), nested);
                            off += plen;
                        }
                        continue;

                    case QUERYKEY:
                        {
                            object[] nested = Msg.NestedBArrayToObjects(payload, off, plen);
                            // Has List<KT> keys, initiator, uid, includeInitator
                            obs[n] = Activator.CreateInstance(typeof(QueryKey<>).MakeGenericType(new Type[] { nested[0].GetType().GetGenericArguments()[0] }), nested);
                            off += plen;
                        }
                        continue;

                    case LIST:
                    case LISTO:
                        {
                            object[] nested = null;
                            object list = null;
                            if (plen != 1)
                            {
                                nested = Msg.NestedBArrayToObjects(payload, off, plen);
                                List<object> tmp = new List<object>();
                                Type theType = typeof(object);
                                foreach (object obj in (IEnumerable)(dynamic)nested[0])
                                {
                                    object[] innerObjs = BArrayToObjects((byte[])obj);
                                    if (innerObjs.Length == 1)
                                    {
                                        theType = innerObjs[0].GetType();
                                        tmp.Add(innerObjs[0]);
                                    }
                                    else
                                        throw new IsisException("internal error: BArrayToObjects<LIST>");
                                }
                                list = (dynamic)Activator.CreateInstance(typeof(List<>).MakeGenericType(new Type[] { conv == LISTO ? typeof(object) : theType }));
                                MethodInfo mi = list.GetType().GetMethod("Add");
                                foreach (object o in tmp)
                                    mi.Invoke(list, new object[] { o });
                            }
                            else
                            {
                                foreach (var sdt in UserDefinedTypesList)
                                    if (sdt.Value == payload[off])
                                    {
                                        list = (dynamic)Activator.CreateInstance(typeof(List<>).MakeGenericType(new Type[] { sdt.Key }));
                                        break;
                                    }
                                if (list == null)
                                    throw new IsisException("List<T>: failed to reconstruct type for code=" + payload[off] + " @ " + Isis.ExtractStackTrace());
                            }
                            obs[n] = list;
                            off += plen;
                        }
                        continue;


                    default:
                        if (UserDefinedTypesTable[conv] == null)
                        {
                            string pcnts = " ";
                            foreach (byte b in payload)
                                pcnts += b.ToString("X2") + " ";
                            Isis.WriteLine("WARNING: Msg.BArrayToObjects was unable to demarshall a byte array... Payload.len = " +
                                payload.Length + ", contents = [" + pcnts + "], start=" + start + ", len=" + len);
                            return new object[0];
                        }

                        // Note: no need to retain the lock; it functions as a barrier in this case because the table entry is write-once
                        byte[] ba = new byte[plen];
                        for (int k = 0; k < plen; k++)
                            ba[k] = payload[off++];
                        if (UserDefinedTypesTable[conv].theMarshaller != null)
                        {
                            if ((IsisSystem.Debug & IsisSystem.CALLBACKS) != 0)
                                Group.ReportCb(UserDefinedTypesTable[conv].theType, ba);
                            obs[n] = UserDefinedTypesTable[conv].theConstructor.Invoke(new object[] { ba });
                        }
                        else
                        {
                            // Automarshaller
                            FieldInfo[] theFields = UserDefinedTypesTable[conv].theFields;
                            obs[n] = UserDefinedTypesTable[conv].theConstructor.Invoke(new object[0]);
                            Object[] values = Msg.BArrayToObjects(ba);
                            for (int k = 0; k < theFields.Length; k++)
                                theFields[k].SetValue(obs[n], values[k]);
                        }
                        break;
                }
            }

            using (new LockAndElevate(cacheLock)) { lastPayloadArg = payload; lastStart = start; lastLen = len; lastObs = obs; lastPayloadLen = payload.Length; }
            return obs;
        }

        private static byte[] sign(byte[] hash)
        {
            int nb = Isis.ISIS_AES.BlockSize >> 3;
            byte[] IV = new byte[nb];
            Isis.ISIS_AESSEED.NextBytes(IV);
            for (int i = 0; i < nb; i++)
                IV[i] = (byte)i;
            return Group.encipher(null, Isis.ISIS_AES, Isis.ISIS_AES_LOCK, IV, hash);
        }

        private static byte[] recomputeSignature(byte[] IV, byte[] hash)
        {
            return Group.encipher(null, Isis.ISIS_AES, Isis.ISIS_AES_LOCK, IV, hash);
        }

        internal static LockObject VerifyLock = new LockObject("VerifyLock", ThreadPriority.Highest);
        internal Group theGroup;

        internal static bool VerifySignature(byte[] payload, int start, int len)
        {
            if (Isis.ISIS_MD5SIGS && (len - start) > Isis.ISIS_MSGPADDING)
            {
                using (new LockAndElevate(VerifyLock))
                using (MemoryStream ms = new MemoryStream(payload, start, len - Isis.ISIS_MSGPADDING))
                {
                    using (HMAC hm = new HMACMD5(new byte[8] { 56, 78, 9, 23, 10, 87, 33, 11 }))
                    {
                        int x = start + len - Isis.ISIS_MSGPADDING;
                        byte[] hash = hm.ComputeHash(ms);
                        if (Isis.ISIS_AES != null)
                        {
                            int nb = Isis.ISIS_AES.BlockSize >> 3;
                            byte[] IV = new byte[nb];
                            for (int i = 0; i < nb; i++)
                                IV[i] = payload[x + i];
                            hash = recomputeSignature(IV, hash);
                        }
                        if (hash.Length != Isis.ISIS_MSGPADDING)
                        {
                            if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                                Isis.WriteLine("Rejecting a message: hash length is incorrect");
                            return false;
                        }
                        for (int i = 0; i < hash.Length; i++)
                            if (payload[x++] != hash[i])
                            {
                                if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                                    Isis.WriteLine("Rejecting a message: signature verification failed");
                                return false;
                            }
                    }
                }
            }
            return true;
        }

        private static byte[] BitsToBytes(bool[] ba)
        {
            int len = 0;
            if (ba != null)
                len = ba.Length;
            byte[] bv = new byte[4 + ((len + 7) >> 3)];
            if (ba == null)
                len = -1;
            bv[0] = (byte)((len >> 24) & 0xFF);
            bv[1] = (byte)((len >> 16) & 0xFF);
            bv[2] = (byte)((len >> 8) & 0xFF);
            bv[3] = (byte)(len & 0xFF);
            if (len <= 0)
                return bv;
            byte[] bits = new byte[] { 1, 2, 4, 8, 16, 32, 64, 128 };
            for (int i = 0; i < ba.Length; i++)
                if (ba[i])
                    bv[(i >> 3) + 4] |= bits[i & 0x7];
            return bv;
        }

        private static bool[] BytesToBits(byte[] bv)
        {
            int bl = ((int)bv[0] << 24) + ((int)bv[1] << 16) + ((int)bv[2] << 8) + ((int)bv[3]);
            if (bl == -1)
                return null;
            bool[] ba = new bool[bl];
            byte[] bits = new byte[] { 1, 2, 4, 8, 16, 32, 64, 128 };
            for (int i = 0; i < bl; i++)
                ba[i] = ((bv[4 + (i >> 3)] & bits[i & 0x7]) != 0);
            return ba;
        }

        // Checks that the view in the group this P2P message is associated with (if any) has 
        // reached or surpassed the viewid mentioned in the message itself
        internal static bool checkView(Msg m)
        {
            if (m.theGroup == null || m.theGroup.theView == null)
                return true;
            using (new LockAndElevate(m.theGroup.ViewLock))
                return m.theGroup.theView.viewid >= m.vid;
        }
    }

    // Currently only needs (and hence offers) a subset of the true Socket interface
    internal class MCMDSocket
    {
        internal const int VIRTUAL = 0;
        internal const int PHYSICAL = 1;

        internal const int UNKNOWN = -1;
        internal const int USEUNICAST = -2;

        internal int mySlot = -1;
        internal bool isTrackingProxy;

        [AutoMarshalled]
        public class GRPair
        {
            public Address gaddr;
            public int rate;

            // Needed for AutoMarshaller
            public GRPair()
            {
            }

            internal GRPair(Address ga, int r)
            {
                gaddr = ga;
                rate = r;
            }
        }

        internal static void Setup(Group g)
        {
            g.doRegisterAggregator<int, GRPair[]>((Aggregator<int, GRPair[]>)delegate(int key, GRPair[] left, GRPair[] down)
            {

                if ((IsisSystem.Debug & IsisSystem.MCMD) != 0)
                {
                    string ll, dl;
                    ll = ""; dl = "";
                    foreach (GRPair grp in left)
                        ll += " (" + grp.gaddr + "::" + grp.rate + ") ";
                    foreach (GRPair grp in down)
                        ll += " (" + grp.gaddr + "::" + grp.rate + ") ";
                    Isis.WriteLine("GRAggregator(key=" + key + "): left list = <" + ll + ">, downlist = <" + dl + ">");
                }
                List<GRPair> Agg = new List<GRPair>();
                int leftIdx = 0, downIdx = 0;
                while (leftIdx < left.Length && downIdx < down.Length)
                {
                    Address la = left[leftIdx].gaddr, da = down[downIdx].gaddr;
                    switch (Math.Sign(la.CompareTo(da)))
                    {
                        case -1:
                            if ((IsisSystem.Debug & IsisSystem.MCMD) != 0)
                                Isis.WriteLine("GRAggregator Retain left=(" + left[leftIdx].gaddr + "::" + left[leftIdx].rate + ")");
                            Agg.Add(left[leftIdx++]);
                            break;
                        case 0:
                            // Unclear here if it really is best to take max this way, as opposed to min or
                            // some kind of average.  Technically speaking, everyone is measuring the same thing.
                            // In fact technically speaking, new values are better than old ones.  But for inner
                            // level, which is fresher: left, or down?  In the end I decided to just use max.

                            if ((IsisSystem.Debug & IsisSystem.MCMD) != 0)
                                Isis.WriteLine("GRAggregator Compute new pair=(" + down[downIdx].gaddr + "::" + Math.Max(left[leftIdx].rate, down[downIdx].rate) + ")");
                            Agg.Add(new GRPair(la, Math.Max(left[leftIdx++].rate, down[downIdx++].rate)));
                            break;
                        case 1:
                            if ((IsisSystem.Debug & IsisSystem.MCMD) != 0)
                                Isis.WriteLine("GRAggregator Retain down=(" + down[downIdx].gaddr + "::" + down[downIdx].rate + ")");
                            Agg.Add(down[downIdx++]);
                            break;
                    }
                }
                while (leftIdx < left.Length)
                {
                    if ((IsisSystem.Debug & IsisSystem.MCMD) != 0)
                        Isis.WriteLine("GRAggregator Retain extra left=(" + left[leftIdx].gaddr + "::" + left[leftIdx].rate + ")");
                    Agg.Add(left[leftIdx++]);
                }
                while (downIdx < down.Length)
                {

                    if ((IsisSystem.Debug & IsisSystem.MCMD) != 0)
                        Isis.WriteLine("GRAggretaor Retain  extra down=(" + down[downIdx].gaddr + "::" + down[downIdx].rate + ")");
                    Agg.Add(down[downIdx++]);
                }
                return Agg.ToArray();
            });
        }

        internal static Thread ReMappingThread;

        internal static void RunMappingTask()
        {
            if (Isis.ISIS_TCP_ONLY)
                return;
            ReMappingThread = new Thread(delegate()
            {
                try
                {
                    while (!IsisSystem.IsisWasActive)
                        Thread.Sleep(1000);
                    while (IsisSystem.IsisActive)
                    {
                        IsisSystem.RTS.ThreadCntrs[26]++;
                        RemapDelay();
                        Isis.ORACLE.Send(Isis.REMAP, Isis.MapperEpochId, WaitForAggRate(Isis.MapperEpochId));
                        Isis.MapperEpochId++;
                    }
                }
                catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                IsisSystem.ThreadTerminationMagic();
            });
            ReMappingThread.Name = "MCMD Remapping task";
            ReMappingThread.Start();
        }

        private static void RemapDelay()
        {
            long waiting = 0;
            while (IsisSystem.IsisActive)
            {
                Thread.Sleep(1000);
                if ((waiting += 1000) > Isis.ISIS_REMAPDELAY)
                    break;
                if (Isis.ORACLE == null || Isis.ISISMEMBERS == null || !Isis.ORACLE.HasFirstView || !Isis.ISISMEMBERS.HasFirstView)
                    continue;
                WaitForAggRate(Isis.MapperEpochId++);
            }
        }

        volatile internal static bool mapInitializationRunning = false;

        internal static void InitializeMap(Address who, string[] gnames, Address[] gaddrs)
        {
            if (mapInitializationRunning)
                return;
            mapInitializationRunning = true;
            while (Isis.ISISMEMBERS == null || !Isis.ISISMEMBERS.HasFirstView)
                Thread.Sleep(25);
            Address[] jlist = Isis.Expand(new Address[] { who });
            foreach (Address j in jlist)
                Isis.ISISMEMBERS.doP2PSend(j, true, Isis.REMAP, MCMDvirtual.GetMapList(gnames, gaddrs));
        }

        internal class MCMDvirtual
        {
            internal int VirtIPAddr;
            internal int PhysIPAddr;
            internal BoundedBuffer inBuf;
            internal int outBufPtr;

            internal MCMDvirtual(int v, int p, BoundedBuffer bb, int lo)
            {
                VirtIPAddr = v;
                PhysIPAddr = p;
                inBuf = bb;
                outBufPtr = lo;
            }

            internal int[] GetMap()
            {
                return new int[2] { VirtIPAddr, PhysIPAddr };
            }

            internal static int[,] GetMapAll()
            {
                int[,] theMap = new int[MappingLen, 2];
                int n = 0;
                using (new LockAndElevate(MappingLock))
                {
                    for (int i = 0; i < MappingLen; i++)
                    {
                        if (theMapping[i] != null)
                        {
                            theMap[n, MCMDSocket.VIRTUAL] = theMapping[i].VirtIPAddr;
                            theMap[n, MCMDSocket.PHYSICAL] = theMapping[i].PhysIPAddr;
                            ++n;
                        }
                    }
                }
                if (n < MappingLen)
                {
                    // This is because my version of Resize only works for 1-dimensional objects
                    int[,] theCopy = new int[n, 2];
                    for (int i = 0; i < n; i++)
                    {
                        theCopy[i, 0] = theMap[i, 0];
                        theCopy[i, 1] = theMap[i, 1];
                    }
                    return theCopy;
                }
                return theMap;
            }

            internal static int[,] GetMapList(string[] gnames, Address[] gaddrs)
            {
                int[] theVirtAddrs = new int[Math.Max(gnames.Length, gaddrs.Length)];
                int next = 0;
                foreach (string gn in gnames)
                {
                    Group tpg = Group.TrackingProxyLookup(gn);
                    if (tpg == null)
                        continue;
                    bool fnd = false;
                    for (int i = 0; i < next && !fnd; i++)
                        if (theVirtAddrs[i] == tpg.myVirtIPAddr)
                            fnd = true;
                    if (fnd)
                        continue;
                    theVirtAddrs[next++] = tpg.myVirtIPAddr;
                }
                foreach (Address ga in gaddrs)
                {
                    Group tpg = Group.TrackingProxyLookup(ga);
                    if (tpg == null)
                        continue;
                    bool fnd = false;
                    for (int i = 0; i < next && !fnd; i++)
                        if (theVirtAddrs[i] == tpg.myVirtIPAddr)
                            fnd = true;
                    if (fnd)
                        continue;
                    theVirtAddrs[next++] = tpg.myVirtIPAddr;
                }
                if (next < theVirtAddrs.Length)
                    Isis.ArrayResize<int>(ref theVirtAddrs, next);
                int[,] theMap = new int[theVirtAddrs.Length, 2];
                next = 0;
                using (new LockAndElevate(MappingLock))
                {
                    for (int i = 0; i < MappingLen; i++)
                        for (int j = 0; j < theVirtAddrs.Length; j++)
                            if (theMapping[i] != null && theMapping[i].VirtIPAddr == theVirtAddrs[j])
                            {
                                theMap[next, MCMDSocket.VIRTUAL] = theMapping[i].VirtIPAddr;
                                theMap[next, MCMDSocket.PHYSICAL] = theMapping[i].PhysIPAddr;
                                ++next;
                                break;
                            }
                }
                return theMap;
            }
        }

        internal class MCMDphysical
        {
            internal int PhysIPAddr;
            internal int refCount;
            internal BoundedBuffer outBuf;
            internal IPAddress outIP;
            internal Thread outThread;
            internal Thread inThread;

            internal MCMDphysical(int pip, BoundedBuffer bb, IPAddress ipa)
            {
                PhysIPAddr = pip;
                refCount = 1;
                outBuf = bb;
                outIP = ipa;
            }
        }

        internal static int MappingLen = 0, MapArrayLen = 10, MapUID = 0;
        internal static MCMDvirtual[] theMapping = new MCMDvirtual[MapArrayLen];
        internal static LockObject MappingLock = new LockObject("MappingLock", ThreadPriority.Highest);

        internal static int nOutBufs = 0;
        internal static MCMDphysical[] outBufs = new MCMDphysical[0];

        internal static string GetState()
        {
            string s = GetMCMDMap() + "\r\nMCMD VIRTUAL SLOTS\r\n";
            using (new LockAndElevate(MappingLock))
            {
                for (int i = 0; i < theMapping.Length; i++)
                    if (theMapping[i] != null)
                        s += "Virtual socket[" + i + "]:  (" + PMCAddr(theMapping[i].VirtIPAddr) + ":" + PMCAddr(theMapping[i].PhysIPAddr) +
                            "), physical-slot [" + theMapping[i].outBufPtr + "], backlog " + theMapping[i].inBuf.fullSlots + "\r\n";
                s += "MCMD PHYSICAL SLOTS\r\n";
                for (int i = 0; i < outBufs.Length; i++)
                    if (outBufs[i] != null && outBufs[i].inThread != null && outBufs[i].outThread != null)
                        s += "Physical socket[" + i + "]: " + PMCAddr(outBufs[i].PhysIPAddr) + ", refcount=" + outBufs[i].refCount +
                            "\r\n  in-thread <" + (outBufs[i].inThread.Name ?? "<Unnamed user thread>") + ">\r\n  out-thread <" +
                            (outBufs[i].outThread.Name ?? "<Unnamed user thread>") + ">, backlog " + outBufs[i].outBuf.fullSlots + "\r\n";
            }
            return s;
        }

        internal static void Shutdown()
        {
            using (new LockAndElevate(MappingLock))
            {
                if (theMapping == null)
                    return;
                for (int i = 0; i < theMapping.Length; i++)
                {
                    MCMDvirtual mv = theMapping[i];
                    theMapping[i] = null;
                    if (mv != null && mv.inBuf != null)
                        BoundedBuffer.unregister(mv.inBuf);
                }
                MappingLen = 0;
                for (int i = 0; i < outBufs.Length; i++)
                {
                    MCMDphysical mp = outBufs[i];
                    if (mp != null && mp.outBuf != null)
                        mp.outBuf.put(null, true);
                }
            }
        }

        // Used when INITIALVIEW is learned in the ORACLE
        internal static void AssignMapInfo(int[,] mms)
        {
            AssignMapInfo(Group.IsisGroups, Group.IsisGroupsLock, mms);
            AssignMapInfo(Group.TPGroups, Group.TPGroupsLock, mms);
            UpdateMCMDMapping(mms);
        }

        private static void UpdateMCMDMapping(int[,] mms)
        {
            using (new LockAndElevate(MappingLock))
                for (int i = 0; i < theMapping.Length; i++)
                    if (theMapping[i] != null && theMapping[i].PhysIPAddr == UNKNOWN)
                        for (int j = 0; j < mms.GetLength(0); j++)
                            if (mms[j, VIRTUAL] == theMapping[i].VirtIPAddr)
                            {
                                theMapping[i].PhysIPAddr = mms[j, PHYSICAL];
                                break;
                            }
        }

        private static void AssignMapInfo(Dictionary<Address, Group> grps, LockObject lo, int[,] mms)
        {
            List<int[]> mmList = new List<int[]>();
            using (new LockAndElevate(lo))
                foreach (var kvp in grps)
                {
                    Group g = kvp.Value;
                    for (int i = 0; i < mms.GetLength(0); i++)
                    {
                        if (g.myVirtIPAddr == 0)
                            g.myVirtIPAddr = Isis.CLASSD + Isis.ISIS_MCRANGE_LOW + Address.GroupNameHash(g.gname);
                        if (g.myVirtIPAddr == mms[i, MCMDSocket.VIRTUAL])
                        {
                            if (g.myPhysIPAddr == mms[i, MCMDSocket.PHYSICAL])
                                break;
                            else if (g.isTrackingProxy)
                            {
                                g.myPhysIPAddr = mms[i, MCMDSocket.PHYSICAL];
                                break;
                            }
                            mmList.Add(new int[] { mms[i, MCMDSocket.VIRTUAL], mms[i, MCMDSocket.PHYSICAL] });
                            break;
                        }
                    }
                }
            foreach (int[] mm in mmList)
                SetMap("AssignMapInfo", "AssignMapInfo", false, mm);
        }

        internal static List<int> OOBAllocation = new List<int>();

        internal static int AllocateAddr()
        {
            using (new LockAndElevate(MappingLock))
            {
                for (int i = 0; i < Isis.OOBMAXIPMCADDRS; i++)
                    if (!OOBAllocation.Contains(Isis.CLASSD + Isis.ISIS_MCRANGE_HIGH + i))
                    {
                        int theAddr = Isis.CLASSD + Isis.ISIS_MCRANGE_HIGH + i;
                        OOBAllocation.Add(theAddr);
                        return theAddr;
                    }
                throw new IsisException("MCMD:OODB->AllocateAddr");
            }
        }

        internal static void DeAllocateAddr(int mcaddr)
        {
            using (new LockAndElevate(MappingLock))
                OOBAllocation.Remove(mcaddr);
        }

        internal MCMDSocket(string gname, bool isTrackingProxy, int VirtIPAddr)
        {
            int PhysIPAddr = MCMDSocket.UNKNOWN;
            using (new LockAndElevate(MappingLock))
            {
                this.isTrackingProxy = isTrackingProxy;
                for (int i = 0; i < MappingLen; i++)
                    if (theMapping[i] != null && theMapping[i].VirtIPAddr == VirtIPAddr)
                    {
                        mySlot = i;
                        return;
                    }
            }
            using (new LockAndElevate(Group.TPGroupsLock))
                foreach (var kvp in Group.TPGroups)
                    if (kvp.Value.myVirtIPAddr == VirtIPAddr)
                    {
                        PhysIPAddr = kvp.Value.myPhysIPAddr;
                        break;
                    }
            SetMap("MCMDSocket(1)", gname, isTrackingProxy, new int[2] { VirtIPAddr, PhysIPAddr });
        }

        internal MCMDSocket(string gname, bool isTrackingProxy, int VirtIPAddr, int PhysIPAddr)
        {
            this.isTrackingProxy = isTrackingProxy;
            SetMap("MCMDSocket(2)", gname, isTrackingProxy, new int[2] { VirtIPAddr, PhysIPAddr });
            using (new LockAndElevate(MappingLock))
            {
                for (int i = 0; i < MappingLen; i++)
                    if (theMapping[i].VirtIPAddr == VirtIPAddr)
                        mySlot = i;
            }
        }

        // Looks up the physical IP address to use for a given slot in the mapping table
        internal IPEndPoint GetRemoteEP()
        {
            using (new LockAndElevate(MappingLock))
                return new IPEndPoint(new IPAddress(theMapping[mySlot].PhysIPAddr & 0xFFFFFFFFL), Isis.ISIS_GROUPPORT);
        }

        // Looks up the physical IP address to use for a given virtual IP address
        // Same information is available in g.myPhysIPAddr
        internal static IPEndPoint GetRemoteEP(Address gaddr)
        {
            int i;
            using (new LockAndElevate(MappingLock))
                if ((i = MCMDLookup(gaddr)) >= 0)
                    return new IPEndPoint(new IPAddress(theMapping[i].PhysIPAddr & 0xFFFFFFFFL), Isis.ISIS_GROUPPORT);
            throw new IsisException("GetRemoteEP: Missing the mapping for virtual address of group " + gaddr);
        }

        private static int MCMDLookup(Address gaddr)
        {
            byte[] hb = gaddr.home.GetAddressBytes();
            long v = ((int)hb[0] << 24) | ((int)hb[1] << 16) | ((int)hb[2] << 8) | (int)hb[3];
            for (int i = 0; i < MappingLen; i++)
                if (theMapping[i] != null && theMapping[i].VirtIPAddr == v)
                    return i;
            return -1;
        }

        internal static int nextPhysIPAddr = 0;

        internal static void SetMap(string where, string gname, bool isTrackingProxy, int[] mMap)
        {
            int VirtIPAddr = mMap[VIRTUAL], PhysIPAddr = mMap[PHYSICAL], firstUnusedSlot = -1;
            using (new LockAndElevate(MappingLock))
            {
                int ListeningOn = -1, mySlot = -1;
                for (int i = 0; i < MappingLen; i++)
                {
                    if (theMapping[i] == null)
                    {
                        firstUnusedSlot = i;
                        continue;
                    }
                    if (theMapping[i].VirtIPAddr == VirtIPAddr)
                    {
                        if ((theMapping[i].PhysIPAddr == PhysIPAddr || PhysIPAddr == UNKNOWN) && theMapping[i].outBufPtr != -1)
                            return;
                        if (mySlot == -1)
                            mySlot = i;
                        else
                            throw new IsisException("SetMap: barf(1)");
                    }
                    if (theMapping[i].PhysIPAddr == PhysIPAddr && theMapping[i].outBufPtr != -1)
                        ListeningOn = theMapping[i].outBufPtr;
                }
                // At this point, we know which mapping slot this group is using, and if there is any physical listener, we know the listener too
                if (mySlot == -1)
                {
                    // Needs a new mapping slot, allocate one.  If we noticed a free slot can use that, else extend the table
                    if ((IsisSystem.Debug & IsisSystem.MCMDMAP) != 0)
                        Isis.WriteLine("SetMap(" + where + "): Installing a new MCMD mapping: " + PMCAddr(VirtIPAddr) + " to " + PMCAddr(PhysIPAddr));
                    mySlot = firstUnusedSlot;
                    if (mySlot == -1)
                    {
                        if (MappingLen == MapArrayLen)
                        {
                            MapArrayLen += 10;
                            Isis.ArrayResize<MCMDvirtual>(ref theMapping, MapArrayLen);
                        }
                        mySlot = MappingLen++;
                    }
                    if (isTrackingProxy)
                        ListeningOn = -1;
                    theMapping[mySlot] = new MCMDvirtual(VirtIPAddr, PhysIPAddr, new BoundedBuffer(gname + ":MCMD", Isis.ISIS_MCMDBBSIZE, ILock.LLBBIN, -1, -1), ListeningOn);
                    if (ListeningOn != -1)
                    {
                        outBufs[ListeningOn].refCount++;
                        return;
                    }
                }
                else if (theMapping[mySlot].PhysIPAddr != PhysIPAddr)
                {
                    // SetMap can change UNKNOWN to some known address and we'll ignore known to UNKNOWN
                    if (PhysIPAddr == UNKNOWN)
                        return;
                    // Shouldn't be called to switch a known address to some other different mapping (ReMap cleans up first)
                    if (theMapping[mySlot].PhysIPAddr != UNKNOWN && theMapping[mySlot].PhysIPAddr != PhysIPAddr)
                        throw new IsisException("In setMap virtual IP addr " + PMCAddr(VirtIPAddr) + " was mapped to " +
                            PMCAddr(theMapping[mySlot].PhysIPAddr) + " remapped to " + PMCAddr(PhysIPAddr));
                    theMapping[mySlot].PhysIPAddr = PhysIPAddr;
                }

                if (isTrackingProxy)
                    // When mapping on behalf of a tracking proxy, no need to actually listen to the group
                    return;

                if (theMapping[mySlot].outBufPtr == -1 && ListeningOn != -1)
                {
                    // Add reference to some existing output mapping
                    theMapping[mySlot].outBufPtr = ListeningOn;
                    outBufs[ListeningOn].refCount++;
                    return;
                }

                // We reach this point if there is no existing output mapping for this (virt:phys) pair and the group is real
                for (int i = 0; i < outBufs.Length; i++)
                    if (outBufs[i] != null && outBufs[i].PhysIPAddr == PhysIPAddr)
                    {
                        Isis.WriteLine("WARNING: In SetMap " + PMCAddr(VirtIPAddr) + ":" + PMCAddr(PhysIPAddr) + "... group <" + gname + "> wasn't initialized");
                        return;
                    }
                if (PhysIPAddr != USEUNICAST && PhysIPAddr != UNKNOWN)
                    StartMCMDThreads(where, PhysIPAddr, mySlot);
            }
        }

        // Called while holding lock on theMapping
        private static void StartMCMDThreads(string where, int PhysIPAddr, int mySlot)
        {
            int myOutbufSlot;
            string mcaddr;
            if (Isis.ISIS_UNICAST_ONLY || Isis.ISIS_TCP_ONLY)
                return;
            if ((IsisSystem.Debug & IsisSystem.MCMDMAP) != 0)
                Isis.WriteLine("SetMap(" + where + "): Starting an MCMD Socket reader and writer for PhysIP " + PMCAddr(PhysIPAddr));
            IPAddress physIP = Isis.LastIPv4(mcaddr = PMCAddr(PhysIPAddr));

            bool fnd = false;
            for (myOutbufSlot = 0; myOutbufSlot < nOutBufs; myOutbufSlot++)
                if (outBufs[myOutbufSlot] == null)
                {
                    fnd = true;
                    break;
                }
            if (fnd == false)
            {
                myOutbufSlot = nOutBufs;
                nOutBufs += 10;
                Isis.ArrayResize<MCMDphysical>(ref outBufs, nOutBufs);
            }
            // Creates an outbuf and sets reference count to 1
            outBufs[myOutbufSlot] = new MCMDphysical(PhysIPAddr, new BoundedBuffer("MCMD-outgoing:" + PMCAddr(PhysIPAddr), Isis.ISIS_MCMDBBSIZE, ILock.LLBBOUT, -1, -1), physIP);
            theMapping[mySlot].outBufPtr = myOutbufSlot;
            Socket theSocket = MCMDSockSetup(physIP);
            outBufs[myOutbufSlot].outThread = new Thread(delegate()
            {
                int sendCount = 0;
                BoundedBuffer bb = outBufs[myOutbufSlot].outBuf;
                try
                {
                    // Sender side
                    while (!IsisSystem.IsisActive)
                        Thread.Sleep(250);
                    IPEndPoint remoteEP = new IPEndPoint(physIP, Isis.ISIS_GROUPPORT);
                    while (IsisSystem.IsisActive)
                    {
                        ++sendCount;
                        IsisSystem.RTS.ThreadCntrs[27]++;
                        byte[] buffer = (byte[])bb.get();
                        if (buffer == null)
                        {
                            using (new LockAndElevate(MappingLock))
                                outBufs[myOutbufSlot] = null;
                            break;
                        }
                        ReliableSender.CheckLen(buffer);
                        long before = Isis.NOW();
                        int bs = 0;
                        try
                        {
                            bs = theSocket.SendTo(buffer, remoteEP);
                        }
                        catch (IOException) { }
                        catch (SocketException) { }
                        long after = Isis.NOW();
                        if ((IsisSystem.Debug & IsisSystem.WARNIFSLOW) != 0 && (after - before) > 250)
                            Isis.WriteLine("WARNING: Sending an IPMC took " + Isis.TimeToString(after - before) + "s");
                        using (new LockAndElevate(IsisSystem.RTS.Lock))
                        {
                            IsisSystem.RTS.IPMCBsent += bs;
                            IsisSystem.RTS.IPMCsent++;
                        }
                    }
                }
                catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                if (IsisSystem.IsisActive)
                {
                    if (theSocket != null)
                        try { theSocket.Close(); }
                        catch { }
                    BoundedBuffer.unregister(bb);
                }
                IsisSystem.ThreadTerminationMagic();
            });
            outBufs[myOutbufSlot].outThread.Name = "ISIS MCMD: send-to " + mcaddr;
            outBufs[myOutbufSlot].outThread.Priority = ThreadPriority.AboveNormal;
            outBufs[myOutbufSlot].outThread.Start();

            outBufs[myOutbufSlot].inThread = new Thread(delegate()
            {
                try
                {
                    while (!IsisSystem.IsisActive)
                        Thread.Sleep(250);
                    int failedReceiveCount = 0;
                    // Receiver side
                    while (IsisSystem.IsisActive)
                    {
                        IsisSystem.RTS.ThreadCntrs[28]++;
                        int[] myVSocks = null;
                        int myMapUID = -1;
                        using (new LockAndElevate(MappingLock))
                        {
                            if (myMapUID != MapUID)
                                if ((myVSocks = setVSocks(PhysIPAddr)) == null)
                                    break;
                            myMapUID = MapUID;
                        }
                        byte[] buffer = new byte[Isis.ISIS_MAXMSGLEN + 1024];
                        int len;
                        try
                        {
                            len = theSocket.Receive(buffer);
                            IsisSystem.RTS.ThreadCntrs[32]++;
                            using (new LockAndElevate(IsisSystem.RTS.Lock))
                            {
                                IsisSystem.RTS.IPMCBrcvd += len;
                                IsisSystem.RTS.IPMCrcvd++;
                            }
                            failedReceiveCount = 0;
                        }
                        catch (IOException) { len = 0; }
                        catch (SocketException) { len = 0; }
                        catch { if (failedReceiveCount++ > 10) break; len = 0; }
                        if (len > Isis.ISIS_MAXMSGLEN)
                            Isis.WriteLine("WARNING (c): MAXMSGLEN set to " + Isis.ISIS_MAXMSGLEN + ", but received length=" + len);
                        if (len > 4)
                        {
                            int VAddr = 0;
                            for (int i = 0; i < 4; i++)
                                VAddr = (VAddr << 8) | buffer[len - 4 + i];
                            if (len != Isis.ISIS_MAXMSGLEN)
                                Isis.ArrayResize<byte>(ref buffer, len);
                            if (!AcceptIncoming(myVSocks, buffer, VAddr))
                            {
                                Group g;
                                List<byte[]> ae = null;
                                if ((g = Group.Lookup(VAddr)) != null && (ae = g.IPMCArrivedEarly) != null)
                                    ae.Add(buffer);
                                else
                                {
                                    if ((IsisSystem.Debug & IsisSystem.DISCARDS) != 0)
                                        Isis.WriteLine("WARNING: Dr. Multicast filtering a incoming multicast: not a member of VirtIP=" + PMCAddr(VAddr));
                                    using (new LockAndElevate(IsisSystem.RTS.Lock))
                                        IsisSystem.RTS.Discarded++;
                                }
                            }
                        }
                    }
                }
                catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                if (IsisSystem.IsisActive)
                {
                    if (outBufs[myOutbufSlot] != null && outBufs[myOutbufSlot].outIP != null)
                    {
                        if ((IsisSystem.Debug & IsisSystem.MCMD) != 0)
                            if (Isis.InterfaceIds == null)
                                try
                                {
                                    if (outBufs[myOutbufSlot] != null)
                                        theSocket.SetSocketOption(SocketOptionLevel.IP, SocketOptionName.DropMembership,
                                            new MulticastOption(outBufs[myOutbufSlot].outIP)); // Leave multicast group
                                }
                                catch (IOException) { /* Just in case */ }
                                catch (ObjectDisposedException) { }
                            else foreach (int id in Isis.InterfaceIds)
                                    try
                                    {
                                        if ((IsisSystem.Debug & IsisSystem.INTERFACES) != 0)
                                            Isis.WriteLine("EXPERIMENTAL CODE: LEAVE MULTICAST INTERFACE " + id);
                                        if (outBufs[myOutbufSlot] != null)
                                            theSocket.SetSocketOption(SocketOptionLevel.IP, SocketOptionName.DropMembership,
                                                new MulticastOption(outBufs[myOutbufSlot].outIP, id)); // Leave multicast group
                                    }
                                    catch (IOException e) { Isis.WriteLine("IP_DropMembership failed: " + e); /* Just in case */ }
                                    catch (ObjectDisposedException) { }
                    }
                }
                IsisSystem.ThreadTerminationMagic();
            });
            outBufs[myOutbufSlot].inThread.Name = "ISIS MCMD: receive-on " + mcaddr;
            outBufs[myOutbufSlot].inThread.Priority = ThreadPriority.Highest;
            outBufs[myOutbufSlot].inThread.Start();
        }

        private static bool AcceptIncoming(int[] myVSocks, byte[] buffer, int VAddr)
        {
            BoundedBuffer bb = null;
            using (new LockAndElevate(MappingLock))
                for (int j = 0; j < myVSocks.Length && bb == null; j++)
                    if (theMapping[myVSocks[j]] != null && VAddr == theMapping[myVSocks[j]].VirtIPAddr)
                        using (new LockAndElevate(Group.IsisGroupsLock))
                            foreach (var kvp in Group.IsisGroups)
                                if (kvp.Value.myVirtIPAddr == VAddr)
                                {
                                    // Found a reverse-mapping slot and confirmed that I'm (still) a member of the group
                                    if (theMapping[myVSocks[j]] != null)
                                        bb = theMapping[myVSocks[j]].inBuf;
                                    break;
                                }
            if (bb != null)
            {
                bb.put(buffer);
                return true;
            }
            return false;
        }

        internal static void drainIPMCArrivedEarly(List<byte[]> ae)
        {
            if (ae == null)
                return;
            int[] AllVsocks = new int[theMapping.Length];
            for (int i = 0; i < theMapping.Length; i++)
                AllVsocks[i] = i;
            foreach (byte[] bb in ae)
            {
                int VAddr = 0;
                for (int i = 0; i < 4; i++)
                    VAddr = (VAddr << 8) | bb[bb.Length - 4 + i];
                AcceptIncoming(AllVsocks, bb, VAddr);
            }
        }

        internal static void DeleteMapping(int VirtIPAddr, int PhysIPAddr)
        {
            int fndIdx = -1;
            using (new LockAndElevate(MappingLock))
                for (int i = 0; i < MappingLen; i++)
                    if (theMapping[i] != null && theMapping[i].VirtIPAddr == VirtIPAddr)
                    {
                        fndIdx = i;
                        break;
                    }
            if (fndIdx != -1)
            {
                MCMDDisposeMapping(fndIdx);
                CheckOutBufs();
            }
        }

        // Caller holds the MappingLock
        internal static int[] setVSocks(int PhysIPAddr)
        {
            int cnt = 0;
            for (int i = 0; i < MappingLen; i++)
                if (theMapping[i] != null && theMapping[i].PhysIPAddr == PhysIPAddr)
                    cnt++;
            if (cnt > 0)
            {
                int[] myVSocks = new int[cnt];
                cnt = 0;
                for (int i = 0; i < MappingLen; i++)
                    if (theMapping[i] != null && theMapping[i].PhysIPAddr == PhysIPAddr)
                        myVSocks[cnt++] = i;
                return myVSocks;
            }
            return null;
        }

        internal bool UseUnicast()
        {
            if (mySlot == -1)
                return true;
            using (new LockAndElevate(MappingLock))
                return theMapping[mySlot] == null || theMapping[mySlot].PhysIPAddr == USEUNICAST;
        }

        internal static void GotMCMDReport(Msg m)
        {
            Object[] obs = Msg.BArrayToObjects(m.payload, typeof(Address[]), typeof(int[]));
            Address[] gaddrs = (Address[])obs[0];
            int[] rates = (int[])obs[1];
            for (int idx = 0; idx < obs.Length; idx++)
            {
                Group g = Group.TrackingProxyLookup(gaddrs[idx]);
                if (g != null)
                {
                    Isis.WriteLine("GotMCMDReport, setting group " + g.gaddr + " rate to " + rates[idx]);
                    g.rcvdMcastsRate = rates[idx];
                }
            }
        }

        internal static void ComputeMCMDMapping(int epochId, GRPair[] grprs)
        {
            if (!Isis.ISIS_UNICAST_ONLY)
            {
                if (epochId > Isis.MapperEpochId)
                {
                    Isis.MapperEpochId = epochId;
                    using (new LockAndElevate(Group.TPGroupsLock))
                        foreach (var kvp in Group.TPGroups)
                        {
                            Group g = kvp.Value;
                            bool fnd = false;
                            foreach (GRPair gpr in grprs)
                                if (g.gaddr.Equals(gpr.gaddr))
                                {
                                    if ((IsisSystem.Debug & IsisSystem.MCMD) != 0)
                                        Isis.WriteLine("MCMD: ComputeMCMDMapping setting group multicast for tracking proxy of <" + g.gname + "> multicast rate to " + gpr.rate);
                                    g.rcvdMcastsRate = gpr.rate;
                                    fnd = true;
                                    break;
                                }
                            if (!fnd)
                                Isis.WriteLine("WARNING: MCMD ComputeMCMDMapping: No rate found for group " + g.gname);
                        }
                    Group[] tpgs;
                    using (new LockAndElevate(Group.TPGroupsLock))
                        tpgs = Group.TPGroups.Values.ToArray();
                    // Groups that asked us to use IPMC will seem to have "infinite" rates hence will bubble to the top
                    // Groups that asked to use Unicast will have "-infinite" rate and will be at the bottom
                    SortByRateTimesSize(tpgs);
                    MergeSimilarGroups(tpgs);
                    AllocateMCResources(tpgs);
                    Isis.AnnounceMCMDMapping();
                    if ((IsisSystem.Debug & IsisSystem.MCMDMAP) != 0)
                        Isis.WriteLine("Computed new MCMD Mapping: " + GetMCMDMap());
                }
            }
        }

        internal class RSInfo : IComparable
        {
            internal int rateTimesSize;
            internal Group gr;
            internal RSInfo(int r, Group g)
            {
                rateTimesSize = r;
                gr = g;
            }

            public int CompareTo(Object what)
            {
                return ((RSInfo)what).rateTimesSize - rateTimesSize;
            }
        }

        internal static void SortByRateTimesSize(Group[] groups)
        {
            RSInfo[] rsi = new RSInfo[groups.Length];
            for (int i = 0; i < groups.Length; i++)
            {
                Group g = groups[i];
                int rate = g.rcvdMcastsRate;
                if (rate != int.MaxValue && rate != int.MinValue)
                    rate = g.rcvdMcastsRate * ((g.theView == null) ? 0 : g.theView.members.Length);
                rsi[i] = new RSInfo(rate, g);
            }
            Array.Sort(rsi);
            for (int i = 0; i < groups.Length; i++)
                groups[i] = rsi[i].gr;
        }

        internal static void MergeSimilarGroups(Group[] tpgs)
        {
            int sCnt, startAt = 0, forcedCnt = 0;
            List<Group> gotLucky = new List<Group>();
            bool smallSet = (tpgs.Length <= Isis.ISIS_MAXIPMCADDRS);
            for (int i = 0; i < tpgs.Length; i++)
            {
                tpgs[i].sameAs = null;
                if (forcedCnt < Isis.ISIS_MAXIPMCADDRS && ((tpgs[i].flags & Group.G_USEIPMC) != 0 || smallSet) && ((tpgs[i].flags & Group.G_USEUNICAST) == 0))
                {
                    ++forcedCnt;
                    tpgs[i].getsMCADDR = true;
                }
                else
                    tpgs[i].getsMCADDR = false;
            }
            sCnt = 0;
            do
            {
                // Gives up after looking at top 10% unless we're getting 50% or better compression.  Stop at end, or at
                // first group that asked us to UseUnicast (hence: those are just not considered at all)
                int endAt = Math.Min(startAt + tpgs.Length / 10, tpgs.Length - 1);
                while (endAt > startAt && tpgs[endAt].rcvdMcastsRate == int.MinValue)
                    --endAt;
                if (endAt - startAt < 2)
                    break;
                for (int i = startAt; i < endAt; i++)
                {
                    foreach (Group lg in gotLucky)
                        if (closeEnough(tpgs[i], lg))
                        {
                            ++sCnt;
                            tpgs[i].sameAs = lg;
                            break;
                        }
                    if (tpgs[i].sameAs == null)
                    {
                        tpgs[i].getsMCADDR = true;
                        gotLucky.Add(tpgs[i]);
                    }
                    else
                        tpgs[i].getsMCADDR = false;
                }
                startAt = endAt;
            }
            while (sCnt > tpgs.Length / 20 && startAt < tpgs.Length);
        }

        // Check for a 50% or better match....
        internal static bool closeEnough(Group a, Group b)
        {
            if (a == b)
                return true;
            View va, vb;
            using (new LockAndElevate(a.ViewLock))
                va = a.theView;
            using (new LockAndElevate(b.ViewLock))
                vb = b.theView;
            if (va == null || vb == null)
                return false;
            if (va.members.Length > 5)
            {
                // Quick rule of thumb: if we'll be getting a 50% match, odds are that 5 randomly picked members have at least 2 matches
                // So check that property and don't do a careful check on the full membership lists (could be very long) unless this simple
                // test is a success.  We do run a risk of failing to merge groups that are actually similar but we save a lot of time this way
                int cnt5 = 0;
                for (int i = 0; i < 5 && cnt5 < 2; i++)
                    if (vb.GetRawRankOf(va.members[i]) != -1)
                        ++cnt5;
                if (cnt5 < 2)
                    return false;
            }
            // Oh well, no choice now except to grind out the full check.  But stop early if we get lucky and hit the 50% threshold quickly
            int cntMatch = 0, threshold = Math.Max(va.members.Length, vb.members.Length) / 2;
            for (int i = 0; i < va.members.Length && cntMatch < threshold; i++)
                if (va.GetRawRankOf(va.members[i]) != -1)
                    ++cntMatch;
            // Merge these groups if they have > 50% common members, else don't
            return cntMatch >= threshold;
        }

        // Allocate new IPMC addresses (not previously used) to at most ISIS_MAXIPMCADDRS groups
        // WARNING: During the switch-over twice ISIS_MAXIPMCADDRS will temporarily be in use
        internal static void AllocateMCResources(Group[] tpgs)
        {
            using (new LockAndElevate(Group.TPGroupsLock))
            {
                foreach (var kvp in Group.TPGroups)
                {
                    Group g = kvp.Value;
                    g.hasPhysMapping = false;
                    g.myPhysIPAddr = USEUNICAST;
                }
                Isis.nPhysAddrsInUse = 0;
                for (int gn = 0; gn < tpgs.Length && Isis.nPhysAddrsInUse < Isis.ISIS_MAXIPMCADDRS; gn++)
                {
                    Group ga = null;
                    foreach (var kvp in Group.TPGroups)
                        if (kvp.Value.gaddr.Equals(tpgs[gn].gaddr))
                            ga = kvp.Value;
                    if (ga != null)
                        if (ga.getsMCADDR)
                        {
                            ga.hasPhysMapping = true;
                            ga.myPhysIPAddr = nextIPMCAddr(Isis.ISIS_MAXIPMCADDRS << 1);
                        }
                        else if (ga.sameAs != null)
                            ga.myPhysIPAddr = ga.sameAs.myPhysIPAddr;
                        else
                            ga.myPhysIPAddr = USEUNICAST;
                }
            }
            if ((IsisSystem.Debug & IsisSystem.MCMDMAP) != 0)
                Isis.WriteLine("After AllocateMCResources " + GetMCMDMap());
        }

        private static int nextIPMCAddr(int limit)
        {
            if (Isis.ISIS_UNICAST_ONLY)
                return USEUNICAST;
            if (nextPhysIPAddr == 0)
                nextPhysIPAddr = Isis.CLASSD + Isis.ISIS_MCRANGE_LOW + 1;
            List<int> inUse = new List<int>();
            inUse.Add(Isis.ORACLE.myPhysIPAddr);
            using (new LockAndElevate(MappingLock))
            {
                foreach (var a in OOBAllocation)
                    inUse.Add(a);
                for (int i = 0; i < MappingLen; i++)
                    if (theMapping[i] != null && !inUse.Contains(theMapping[i].PhysIPAddr))
                        inUse.Add(theMapping[i].PhysIPAddr);
            }
            using (new LockAndElevate(Group.TPGroupsLock))
                foreach (var kvp in Group.TPGroups)
                {
                    Group g = kvp.Value;
                    if (g.myPhysIPAddr != UNKNOWN && g.myPhysIPAddr != USEUNICAST && !inUse.Contains(g.myPhysIPAddr))
                        inUse.Add(g.myPhysIPAddr);
                }
            for (int a = Isis.CLASSD + Isis.ISIS_MCRANGE_LOW; a < nextPhysIPAddr; a++)
                if (!inUse.Contains(a))
                {
                    IPMCReport(inUse, "successfully assigned", a);
                    return a;
                }
            if (Isis.nPhysAddrsInUse == limit)
            {
                IPMCReport(inUse, "all slots are inuse, hit limit, returning ", USEUNICAST);
                return USEUNICAST;
            }
            nextPhysIPAddr++;
            Isis.nPhysAddrsInUse++;
            IPMCReport(inUse, "all slots are inuse, allocated new address ", nextPhysIPAddr);
            if (nextPhysIPAddr == Isis.CLASSD + Isis.ISIS_MCRANGE_HIGH)
                nextPhysIPAddr = Isis.CLASSD + Isis.ISIS_MCRANGE_LOW;
            return nextPhysIPAddr;
        }

        private static void IPMCReport(List<int> inUse, string s, int a)
        {
            if ((IsisSystem.Debug & IsisSystem.MCMDMAP) != 0)
            {
                string inuse = " ";
                foreach (int x in inUse)
                    inuse += PMCAddr(x) + " ";
                Isis.WriteLine("nextIPMCAddr: InUse { " + inuse + " }, " + s + "  " + PMCAddr(a));
            }
        }

        private static string GetMCMDMap()
        {
            List<Group> mapList = Group.IsisAllGroupsClone(true);
            string res = "MCMD Mapping:\r\n";
            foreach (Group g in mapList)
            {
                res += "    Group [" + g.gname + "]: rate = " + g.rcvdMcastsRate + "    VirtIPAddr=" + PMCAddr(g.myVirtIPAddr) + ", ";
                if (g.getsMCADDR)
                    res += "has a physical IP multicast address: " + PMCAddr(g.myPhysIPAddr);
                else if (g.sameAs != null)
                    res += "shares physical IP multicast address: " + PMCAddr(g.myPhysIPAddr) + " with group [" + g.sameAs.gname + "]";
                else
                    res += "mapped to " + PMCAddr(g.myPhysIPAddr);
                res += "\r\n";
            }
            return res;
        }

        internal static string PMCAddr(int theAddr)
        {
            if (theAddr == USEUNICAST)
                return "<UNICAST>";
            if (theAddr == UNKNOWN)
                return "<UNKNOWN>";
            return ((theAddr >> 24) & 0xFF).ToString() + "." + ((theAddr >> 16) & 0xFF).ToString() + "." + ((theAddr >> 8) & 0xFF).ToString() + "." + (theAddr & 0xFF).ToString();
        }

        internal static Socket MCMDSockSetup(IPAddress theIPAddr)
        {
            if (Isis.ISIS_TCP_ONLY)
                throw new IsisException("MCMDSockSetup called in TCP_ONLY mode");
            Socket theSocket = null;
            try
            {
                theSocket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
                IPEndPoint localEndPoint = new IPEndPoint(IPAddress.Any, Isis.ISIS_GROUPPORT);
                theSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, 1);                      // Allows sharing of this socket by multiple processes on this machine
                theSocket.Bind(localEndPoint);
                theSocket.SetSocketOption(SocketOptionLevel.IP, SocketOptionName.MulticastTimeToLive, Isis.ISIS_TTL);  // Sets TTL (default: to LAN only)
                if (Isis.ISIS_UDPCHKSUM)
                    theSocket.SetSocketOption(SocketOptionLevel.Udp, SocketOptionName.ChecksumCoverage, 1);
                if ((IsisSystem.Debug & IsisSystem.MCMD) != 0)
                    Isis.WriteLine("Physical subscribe to IP address " + theIPAddr);
                if (Isis.InterfaceIds.Length == 0)
                    theSocket.SetSocketOption(SocketOptionLevel.IP, SocketOptionName.AddMembership, new MulticastOption(theIPAddr));
                else foreach (int id in Isis.InterfaceIds)
                        try
                        {
                            if ((IsisSystem.Debug & IsisSystem.INTERFACES) != 0)
                                Isis.WriteLine("EXPERIMENTAL CODE: JOIN MULTICAST INTERFACE " + id);
                            theSocket.SetSocketOption(SocketOptionLevel.IP, SocketOptionName.AddMembership,
                                new MulticastOption(theIPAddr, id)); // Join multicast group
                        }
                        catch (IOException e) { Isis.WriteLine("IP_AddMembership failed: " + e); /* Just in case */ }
                        catch (ObjectDisposedException) { }
                try
                {
                    theSocket.ReceiveBufferSize = (int)Math.Min(64 * Isis.ISIS_MAXMSGLEN, int.MaxValue);
                    theSocket.SendBufferSize = (int)Math.Min(32 * Isis.ISIS_MAXMSGLEN, int.MaxValue);
                    if (!Isis.ISIS_TCP_ONLY)
                        theSocket.ReceiveTimeout = 1000;
                }
                catch (Exception e)
                {
                    Isis.WriteLine("MCMD SockSetup unable to set send/recieve/ReceiveTimeout: " + e);
                }
                return theSocket;
            }
            catch (Exception e)
            {
                if (theSocket != null)
                    theSocket.Close();
                throw new IsisException("MCMD SockSetup failed: error <" + e + "> " + GetMCMDMap());
            }
        }

        internal static int[] GetMap(Address gaddr, bool allocateMapIfUnknown)
        {
            int v;
            using (new LockAndElevate(MappingLock))
            {
                v = AddressToVirtual(gaddr);
                for (int i = 0; i < MappingLen; i++)
                    if (theMapping[i] != null && theMapping[i].VirtIPAddr == v && theMapping[i].PhysIPAddr != UNKNOWN)
                    {
                        if ((IsisSystem.Debug & IsisSystem.MCMDMAP) != 0)
                            Isis.WriteLine("GetMap: MCMD mapping was already known: " + PMCAddr(v) + " to " + PMCAddr(theMapping[i].PhysIPAddr));
                        return theMapping[i].GetMap();
                    }
            }
            Group g = Group.TrackingProxyLookup(gaddr);
            if (!g.hasPhysMapping)
            {
                using (new LockAndElevate(Group.TPGroupsLock))
                    foreach (var kvp in Group.TPGroups)
                    {
                        Group gb = kvp.Value;
                        if (gb.hasPhysMapping && closeEnough(g, gb))
                        {
                            g.hasPhysMapping = true;
                            g.myPhysIPAddr = gb.myPhysIPAddr;
                            if ((IsisSystem.Debug & IsisSystem.MCMDMAP) != 0)
                                Isis.WriteLine("GetMap: Reusing an existing MCMD mapping: " + PMCAddr(v) + " to " + PMCAddr(g.myPhysIPAddr));
                            return new int[2] { v, g.myPhysIPAddr };
                        }
                        else if ((IsisSystem.Debug & IsisSystem.MCMDMAP) != 0)
                            Isis.WriteLine("GetMap: don't compare against " + gb.gname + ": (g == gp)=" + (g == gb) + ", gb.hasPhysMapping=" + gb.hasPhysMapping + ", closeEnough(g, gb)=" + (g != gb && gb.hasPhysMapping ? "false" : "not tested"));
                    }
                if (allocateMapIfUnknown)
                {
                    g.hasPhysMapping = true;
                    g.myPhysIPAddr = nextIPMCAddr(Isis.ISIS_MAXIPMCADDRS);
                    if ((IsisSystem.Debug & IsisSystem.MCMDMAP) != 0)
                        Isis.WriteLine("GetMap: Creating a new MCMD mapping: " + PMCAddr(v) + " to " + PMCAddr(g.myPhysIPAddr));
                }
            }
            return new int[2] { v, g.myPhysIPAddr };
        }

        private static int AddressToVirtual(Address gaddr)
        {
            byte[] hb = gaddr.home.GetAddressBytes();
            int v = ((int)hb[0] << 24) | ((int)hb[1] << 16) | ((int)hb[2] << 8) | (int)hb[3];
            return v;
        }

        internal static int[,] GetMap(Address[] gaddrs)
        {
            int[,] mms = new int[gaddrs.Length, 2];
            for (int i = 0; i < gaddrs.Length; i++)
            {
                int[] mm = GetMap(gaddrs[i], false);
                mms[i, VIRTUAL] = mm[VIRTUAL];
                mms[i, PHYSICAL] = mm[PHYSICAL];
            }
            if ((IsisSystem.Debug & IsisSystem.MCMDMAP) != 0)
            {
                Isis.WriteLine("GetMap[,]: Returning a new MCMD mapping table:");
                for (int i = 0; i < mms.GetLength(0); i++)
                    Isis.WriteLine("   Map virtual address " + PMCAddr(mms[i, VIRTUAL]) + " to physical address " + PMCAddr(mms[i, PHYSICAL]));
            }
            return mms;
        }

        internal static void ReMap(int epochId, int[,] mms)
        {
            bool disposed = false;
            int mmsLen = mms.GetLength(0);
            if ((IsisSystem.Debug & IsisSystem.MCMDMAP) != 0)
            {
                Isis.WriteLine("Received a new MCMD mapping table for epoch(" + epochId + "):");
                for (int i = 0; i < mms.GetLength(0); i++)
                    Isis.WriteLine("   Map virtual address " + PMCAddr(mms[i, VIRTUAL]) + " to physical address " + PMCAddr(mms[i, PHYSICAL]));
            }
            // Clean the mms mapping to eliminate entries that aren't relevant to this process
            int newLen = 0;
            for (int i = 0; i < mms.GetLength(0); i++)
            {
                bool fnd = false;
                using (new LockAndElevate(Group.IsisGroupsLock))
                    foreach (var kvp in Group.IsisGroups)
                        if (kvp.Value.myVirtIPAddr == mms[i, VIRTUAL] && (kvp.Value != Isis.ORACLE || Isis.ClientOf == null))
                            fnd = true;
                if (fnd)
                {
                    mms[newLen, VIRTUAL] = mms[i, VIRTUAL];
                    mms[newLen, PHYSICAL] = mms[i, PHYSICAL];
                    ++newLen;
                }
            }
            if (newLen != mmsLen)
            {
                int[,] newMMS = new int[newLen, 2];
                for (int i = 0; i < newLen; i++)
                {
                    newMMS[i, VIRTUAL] = mms[i, VIRTUAL];
                    newMMS[i, PHYSICAL] = mms[i, PHYSICAL];
                }
                mms = newMMS;
                mmsLen = newLen;
            }
            Dictionary<int, int> QuickMap = new Dictionary<int, int>(64);
            for (int i = 0; i < mmsLen; i++)
                // Make a quick note of the mapping for each virtual address
                QuickMap.Add(mms[i, VIRTUAL], mms[i, PHYSICAL]);

            // Use lists to avoid calls from inside the locked code block
            List<int> disposeEm = new List<int>();
            List<int[]> mapEm = new List<int[]>();
            using (new LockAndElevate(MappingLock))
            {
                // Install the new map, then get everyone to switch to it
                for (int i = 0; i < MappingLen; i++)
                {
                    bool foundIt = false;
                    if (theMapping[i] != null && QuickMap.ContainsKey(theMapping[i].VirtIPAddr))
                    {
                        foundIt = true;
                        break;
                    }
                    if (!foundIt)
                    {
                        disposed = true;
                        disposeEm.Add(i);
                    }
                }
                for (int i = 0; i < mmsLen; i++)
                {
                    int vaddr;
                    for (int j = 0; j < MappingLen; j++)
                        if (theMapping[j] != null && QuickMap.TryGetValue(theMapping[j].VirtIPAddr, out vaddr) && vaddr == mms[i, VIRTUAL])
                        {
                            if (theMapping[j].PhysIPAddr != mms[i, PHYSICAL])
                            {
                                if (theMapping[j].PhysIPAddr != UNKNOWN && theMapping[j].PhysIPAddr != USEUNICAST)
                                {
                                    disposed = true;
                                    disposeEm.Add(j);
                                }
                                mapEm.Add(new int[2] { mms[i, VIRTUAL], mms[i, PHYSICAL] });
                                break;
                            }
                            break;
                        }
                }
            }
            foreach (int i in disposeEm)
                MCMDDisposeMapping(i);
            if (disposed)
                CheckOutBufs();
            foreach (int[] mm in mapEm)
                SetMap("ReMap", "ReMap", false, mm);

            if (epochId == -1)
                return;
            using (new LockAndElevate(Group.IsisGroupsLock))
                foreach (var kvp in Group.IsisGroups)
                {
                    Group g = kvp.Value;
                    int paddr = g.myPhysIPAddr;
                    QuickMap.TryGetValue(g.myVirtIPAddr, out paddr);
                    g.myPhysIPAddr = paddr;
                }
            GRPair[] myGRPairs = Group.GroupRates();
            if ((IsisSystem.Debug & IsisSystem.MCMD) != 0)
            {
                string ll;
                ll = "";
                foreach (GRPair grp in myGRPairs)
                    ll += " (" + grp.gaddr + "::" + grp.rate + ") ";
                Isis.WriteLine("MCMD: Setting GRAggregator(key=" + epochId + "): list = <" + ll + ">)");
            }
            Isis.ISISMEMBERS.SetAggregatorValue<int, GRPair[]>(epochId, myGRPairs);
        }

        internal static GRPair[] WaitForAggRate(int epochId)
        {
            GRPair[] grprs;
            try
            {
                if ((IsisSystem.Debug & IsisSystem.MCMD) != 0)
                    Isis.WriteLine("MCMD: WaitForAggRate(key=" + epochId + ")");
                grprs = Isis.ISISMEMBERS.GetAggregatorResult<int, GRPair[]>(epochId);
                if (grprs == null)
                    return null;
            }
            catch (AggregationFailed)
            {
                if ((IsisSystem.Debug & IsisSystem.MCMD) != 0)
                    Isis.WriteLine("MCMD: Rate aggregator failed.");
                return null;
            }

            if ((IsisSystem.Debug & IsisSystem.MCMD) != 0)
            {
                string ll;
                ll = "";
                foreach (GRPair grp in grprs)
                    ll += " (" + grp.gaddr + "::" + grp.rate + ") ";
                Isis.WriteLine("MCMD: After wait, obtained GRAggregator(key=" + epochId + "): list = <" + ll + ">)");
            }
            return grprs;
        }

        private static void MCMDDisposeMapping(int n)
        {
            using (new LockAndElevate(MappingLock))
            {
                // Dispose of the mapping entry and associated input buffer
                if (theMapping[n] == null)
                    return;
                BoundedBuffer.unregister(theMapping[n].inBuf);
                if (theMapping[n].outBufPtr != -1)
                    outBufs[theMapping[n].outBufPtr].refCount--;
                theMapping[n] = null;
            }
        }

        private static void CheckOutBufs()
        {
            // This reduced the number of references to the associated output
            // buffer.  Dispose of it if this was the last reference
            using (new LockAndElevate(MappingLock))
            {
                for (int i = 0; i < nOutBufs; i++)
                    if (outBufs[i] != null)
                        outBufs[i].refCount = 0;
                for (int i = 0; i < MappingLen; i++)
                    if (theMapping[i] != null && theMapping[i].outBufPtr != -1)
                        outBufs[theMapping[i].outBufPtr].refCount++;
                for (int i = 0; i < nOutBufs; i++)
                    if (outBufs[i] != null && outBufs[i].refCount == 0)
                        outBufs[i].outBuf.put(null);
            }
        }

        internal static void UnMap(Address[] gaddrs)
        {
            bool disposed = false;
            // Close listed groups, terminate their mappings
            foreach (Address gaddr in gaddrs)
            {
                int i;
                if ((i = MCMDLookup(gaddr)) != -1)
                {
                    disposed = true;
                    MCMDDisposeMapping(i);
                }
            }
            if (disposed)
                CheckOutBufs();
        }

        internal bool NBSendTo(byte[] buffer)
        {
            return SendTo(buffer, false);
        }

        internal bool SendTo(byte[] buffer)
        {
            return SendTo(buffer, true);
        }

        internal bool SendTo(byte[] buffer, bool OkToBlock)
        {
            long v;
            int n = buffer.Length;
            ReliableSender.CheckLen(buffer);
            BoundedBuffer bb;
            int[] mm = null;
            using (new LockAndElevate(MappingLock))
                if (theMapping[mySlot] == null)
                    return false;
                else if (theMapping[mySlot].outBufPtr == -1)
                    mm = new int[] { theMapping[mySlot].VirtIPAddr, theMapping[mySlot].PhysIPAddr };
            if (mm != null)
                SetMap("SendTo", "SendTo", false, mm);
            using (new LockAndElevate(MappingLock))
            {
                if (theMapping[mySlot] == null || theMapping[mySlot].outBufPtr == -1)
                {
                    if ((IsisSystem.Debug & IsisSystem.MCMD) != 0)
                        Isis.WriteLine("WARNING: Discarding an outgoing message in SendTo: slot " + mySlot + " has no mapping");
                    return false;
                }
                v = theMapping[mySlot].VirtIPAddr;
                bb = outBufs[theMapping[mySlot].outBufPtr].outBuf;
            }
            // Before sending, append 4-byte virtual address as a trailer
            Isis.ArrayResize<byte>(ref buffer, n + 4);
            for (int i = 0; i < 4; i++)
            {
                buffer[n + 3 - i] = (byte)(v & 0xFF);
                v >>= 8;
            }
            if (!OkToBlock && bb.putWillBlock(1))
                return false;
            bb.put(buffer);
            return true;
        }

        internal byte[] Receive()
        {
            while (!IsisSystem.IsisActive)
                Thread.Sleep(250);
            BoundedBuffer bb;
            while (IsisSystem.IsisActive)
            {
                IsisSystem.RTS.ThreadCntrs[29]++;
                using (new LockAndElevate(MappingLock))
                {
                    if (theMapping[mySlot] == null)
                    {
                        // Race condition can arise when starting up and unsure if I'll be an ORACLE member or not...
                        Thread.Sleep(50);
                        return null;
                    }
                    bb = theMapping[mySlot].inBuf;
                }
                byte[] buffer = (byte[])bb.get();
                if (buffer != null)
                {
                    // Strip trailer, then pass up
                    Isis.ArrayResize<byte>(ref buffer, buffer.Length - 4);
                    if (!Msg.VerifySignature(buffer, 0, buffer.Length))
                    {
                        Thread.Sleep(50);
                        continue;
                    }
                }
                return buffer;
            }
            throw new IsisShutdown("Isis Inactive");
        }
    }

    internal class ILock : IDisposable
    {
        internal class LockInfo
        {
            internal LockInfo Next;
            internal int LockLevel;
            internal int LockId;
            internal bool WaitingFor = false;
            internal bool ReLocked = false;

            internal LockInfo(int ll, int li)
            {
                LockLevel = ll;
                LockId = li;
            }

            internal LockInfo(int ll, int li, bool wf)
            {
                LockLevel = ll;
                LockId = li;
                WaitingFor = wf;
            }

            public override string ToString()
            {
                return PLock(LockLevel, LockId) + (ReLocked ? "(relocked)" : "");
            }
        }

        internal class LockInfoHead
        {
            internal LockInfo First;
            internal volatile string Tname;

            internal LockInfoHead(string name)
            {
                Tname = name;
            }
        }

        internal class PWaitInfo
        {
            internal Address gaddr;
            internal Address[] plist;

            internal PWaitInfo(Address g, Address[] p)
            {
                gaddr = g;
                plist = p;
            }
        }

        internal class IdAndCtr
        {
            internal int Id;
            internal int Cntr;

            internal IdAndCtr(int i)
            {
                Id = i;
                Cntr = 1;
            }
        }

        private static int LILEN = 100;
        private static LockObject LInfoLock = new LockObject(false, "LInfoLock");
        private static int next = 0;
        private static LockInfoHead[] LInfo = new LockInfoHead[LILEN];
        private static PWaitInfo[] ProcessWait = new PWaitInfo[LILEN];
        private static LockObject ProcessWaitLock = new LockObject("ProcessWaitLock");
        private static Semaphore[] Barriers = new Semaphore[LILEN];
        private static LockObject BarriersLock = new LockObject("BarriersLock");

        public const int LLBRIEF = 0;
        public const int LLENTRY = 1;
        public const int LLWAIT = 2;
        public const int LLWAITG = 3;
        public const int LLREPLY = 4;
        public const int LLINITV = 5;
        public const int LLLB = 6;
        public const int LLBBOUT = 7;
        public const int LLBBIN = 8;
        public const int LLLARGE = 9;
        public const int LLLARGEBW = 10;
        public const int LLACKB = 11;
        public const int LLP2PB = 12;
        public const int LLFRAG = 13;
        public const int LLIPMC = 14;
        public const int LLDELIVERY = 15;
        public const int LLTCP = 16;

        private const int NLEVELS = 17;

        /* Sub-ids used with LLWAIT */
        public const int LCLIENTOF = 0;
        public const int LGVEUPDATE = 1;
        public const int LCOMMIT = 2;
        public const int LFLUSH = 3;
        public const int LFLOWCNTRL = 4;
        public const int LFLOWCNTRLB = 5;
        public const int LJOINB = 6;

        private static Semaphore[][] Semaphores = new Semaphore[NLEVELS][];
        internal static LockObject SemaphoresLock = new LockObject(false, "SemaphoresLock");
        private static bool[][] isBarrier = new bool[NLEVELS][];
        private static int[][] nWaiters = new int[NLEVELS][];
        private static int[][] Holding = new int[NLEVELS][];
        private static Dictionary<int, IdAndCtr>[] LockIds = new Dictionary<int, IdAndCtr>[NLEVELS];
        private static LockObject LockIdsLock = new LockObject("LockIdsLock", ThreadPriority.Highest);

        private static string[] levels = new string[] { "brief", "entry", "general wait", "join-wait", "wants replies", "needs initial view", "barrier", "bb-out", "bb-in", "large", "largebw", "ackb", "p2pB", "frag", "ipmc", "delivery", "tcp" };
        private static string[] sls = new string[] { "client-of", "gveupdate", "commit", "flush", "flow-control (local backlog)", "flow-control (remote backlog)", "waiting for a join to complete" };

        internal static string PLock(ILock il)
        {
            if (il == null)
                return "<null>";
            return PLock(il.level, il.lockId);
        }

        internal static string PLock(int level, int sublevel)
        {
            if (level < 0 || level >= levels.Length)
                return "<Unknown level: " + level + ">";
            string sl = null;
            if (level == LLWAIT)
            {
                if (sublevel < 0 || sublevel >= sls.Length)
                    sl = "<" + sublevel + ">";
                else
                    sl = sls[sublevel];
            }
            else if (level != LLREPLY && level != LLLB && level != LLBBOUT && level != LLBBIN)
            {
                if (LockIds[level] != null)
                    foreach (KeyValuePair<int, IdAndCtr> kvp in LockIds[level])
                        if (kvp.Value.Id == sublevel)
                        {
                            using (new LockAndElevate(Group.IsisGroupsLock))
                                foreach (var gkvp in Group.IsisGroups)
                                    if (gkvp.Value.gaddr.GetHashCode() == kvp.Key)
                                    {
                                        sl = gkvp.Value.gname;
                                        break;
                                    }
                            if (sl != null)
                                break;
                            using (new LockAndElevate(Group.TPGroupsLock))
                                foreach (var gkvp in Group.TPGroups)
                                    if (gkvp.Value.gaddr.GetHashCode() == kvp.Key)
                                    {
                                        sl = gkvp.Value.gname;
                                        break;
                                    }
                            break;
                        }
                if (sl == null)
                    sl = sublevel.ToString();
            }
            else
                sl = sublevel.ToString();
            return "[" + levels[level] + ":" + sl + "]";
        }

        internal static void Shutdown()
        {
            using (new LockAndElevate(SemaphoresLock))
            {
                for (int i = 0; i < Semaphores.Length; i++)
                    if (Semaphores[i] == null)
                        continue;
                    else for (int j = 0; j < Semaphores[i].Length; j++)
                            if (Semaphores[i][j] != null)
                            {
                                if ((IsisSystem.Debug & IsisSystem.LOCKSTATE) != 0)
                                    Isis.WriteLine("Isis.Shutdown releasing-all for semaphore[" + i + "][" + j + "]");
                                int nr;
                                nr = Math.Max(nWaiters[i][j], 1);
                                Semaphores[i][j].Release(nr);
                            }
            }
        }

        public static string GetState()
        {
            string s = LockObject.GetState() + "THREAD ILOCK STATE:\r\n";
            using (new LockAndElevate(IsisSystem.RTS.Lock))
            {
                if (IsisSystem.RTS.ackProcessingBeganAt > 0)
                    s += "  Thead[*](Ack-processing thread) is processing a received ack/nack for " + Isis.TimeToString(Isis.NOW() - IsisSystem.RTS.ackProcessingBeganAt) + " secs\r\n";
                else
                    s += "  Thead[*](Ack-processing thread) is waiting to remove an object from AckBB\r\n";
                if (IsisSystem.RTS.rcvProcessingBeganAt > 0)
                    s += "  Thead[*](Receive-processing thread) has been running for " + Isis.TimeToString(Isis.NOW() - IsisSystem.RTS.rcvProcessingBeganAt) + " secs\r\n";
                else
                    s += "  Thead[*](Receive-processing thread) is waiting to remove an object on RecvBB\r\n";
            }
            using (new LockAndElevate(SemaphoresLock))
            {
                for (int id = 0; id < LILEN; id++)
                {
                    LockInfo li;
                    using (new LockAndElevate(LInfoLock))
                    {
                        if (LInfo[id] == null || (li = LInfo[id].First) == null)
                            continue;
                        s += "  Thread[" + id + "] (" + (LInfo[id].Tname ?? "Unnamed user-created thread") + ")";
                        if (li.WaitingFor)
                        {
                            s += ": Waiting for " + li;
                            li = li.Next;
                        }
                        if (ProcessWait[id] != null)
                        {
                            if (ProcessWait[id].gaddr != null)
                                s += "; while watching Group " + ProcessWait[id].gaddr + ", members ";
                            s += Address.VectorToString(ProcessWait[id].plist);
                        }
                        if (li != null)
                        {
                            s += "; Holding ";
                            while (li != null)
                            {
                                s += "{" + li + "}";
                                li = li.Next;
                            }
                        }
                    }
                    s += "\r\n";
                }

                s += "SEMAPHORE STATE:\r\n";
                for (int l = 0; l < NLEVELS; l++)
                    if (Semaphores[l] == null)
                        continue;
                    else
                    {
                        for (int id = 0; id < Semaphores[l].Length; id++)
                        {
                            string h = "";
                            if (nWaiters[l][id] == 0 && Holding[l][id] == 0)
                                continue;
                            if (isBarrier[l][id])
                                s += "  (barrier)";
                            else if (l == LLLB || l == LLBBOUT || l == LLBBIN)
                                s += "  (bounded buffer)";
                            else if (Holding[l][id] != 0)
                                h = ", Nholding " + Holding[l][id];
                            s += "  " + PLock(l, id) + ", Nwaiters = " + nWaiters[l][id];
                            s += h + "\r\n";
                        }
                    }
            }
            return s;
        }

        private static Dictionary<int, int> Waiting = new Dictionary<int, int>(100);

        internal int level;
        internal int lockId;
        private int myHashCode = -1;
        private int myIndex = -1;                       // Value meaningful only on ILock and ILockref, which have one owner -- NOT used for barriers and bounded buffers...
        private static Int64 quickPick = 0;

        // Track whether Dispose has been called.
        private bool disposed = false;

        // This class constructor is used only to create an ILockRef.
        private ILock(int level, int lockId, bool grabLock)
        {
            SInit(level, lockId, 1);
            if (grabLock)
                Lock();
        }

        // ... and this one is used only from the barrier code
        private ILock(int level, int lockId, bool grabLock, int init)
        {
            SInit(level, lockId, init);
            if (grabLock)
                Lock();
        }

        // Used only in "using(new ILock(...)) statements.  In such cases the lock release
        // occurs on Dispose, which involves some slightly special handling
        private bool insideUsing = false;

        internal ILock(int level, Address gaddr)
        {
            if (gaddr != null)
            {
                myHashCode = gaddr.GetHashCode();
                int lockId = GetLockId(level, myHashCode);
                SInit(level, lockId, 1);
                insideUsing = true;
                Lock();
            }
            if ((IsisSystem.Debug & IsisSystem.LOCKCHECK) != 0)
                QuickCheck();
        }

        // Again, used only from the barrier logic
        private ILock(int level, Address gaddr, bool grabLock, int init)
        {
            if (gaddr != null)
            {
                myHashCode = gaddr.GetHashCode();
                int lockId = GetLockId(level, myHashCode);
                SInit(level, lockId, init);
                if (grabLock)
                    Lock();
            }
        }

        // Checks for lock cycles but slow for large systems so needs to be eliminated eventually
        private void QuickCheck()
        {
            bool[,] hflag = new bool[16, 4096];

            using (new LockAndElevate(SemaphoresLock))
            {
                for (int id = 0; id < LILEN; id++)
                {
                    LockInfo li;
                    using (new LockAndElevate(LInfoLock))
                    {
                        if (LInfo[id] == null || (li = LInfo[id].First) == null)
                            continue;
                        while (li != null)
                        {
                            int l = li.LockLevel;
                            int k = li.LockId;
                            if (!li.ReLocked && !li.WaitingFor)
                            {
                                if (!hflag[l, k])
                                    hflag[l, k] = true;
                                else
                                    throw new IsisException("Double locked lock!!! Lock id is (" + l + "," + k + ")" + IsisSystem.GetState());
                            }
                            li = li.Next;
                        }
                    }
                }

            }
        }

        private void SInit(int level, int lockId, int init)
        {
            if (level < 0 || level > NLEVELS) throw new ArgumentOutOfRangeException("ILock");
            this.level = level;
            this.lockId = lockId;
            using (new LockAndElevate(SemaphoresLock))
            {
                int n = 1;
                while (n <= lockId)
                    n <<= 1;
                if (Semaphores[level] == null)
                {
                    Semaphores[level] = new Semaphore[n];
                    nWaiters[level] = new int[n];
                    Holding[level] = new int[n];
                    isBarrier[level] = new bool[n];
                }
                else if (Semaphores[level].Length <= n)
                {
                    Isis.ArrayResize<Semaphore>(ref Semaphores[level], n);
                    Isis.ArrayResize<bool>(ref isBarrier[level], n);
                    Isis.ArrayResize<int>(ref nWaiters[level], n);
                    Isis.ArrayResize<int>(ref Holding[level], n);
                }
                if (Semaphores[level][lockId] == null)
                    Semaphores[level][lockId] = new Semaphore(init, Int32.MaxValue);
            }
        }

        internal static ILock IlockRef(int level, int lockId)
        {
            return new ILock(level, lockId, false);
        }

        internal static ILock IlockRef(int level, Address gaddr)
        {
            return new ILock(level, gaddr, false, 0);
        }

        internal static ILock IlockResetAndRef(int level, int lockId, bool grabLock, int init)
        {
            using (new LockAndElevate(SemaphoresLock))
            {
                if (level >= Semaphores.Length || lockId == -1)
                    throw new IsisException("IlockResetAndRef(" + level + ", " + lockId + ", " + grabLock + ", " + init + "): len too large or id<0!");
                if (Semaphores[level] != null && Semaphores[level].Length > lockId)
                    Semaphores[level][lockId] = null;
            }
            return new ILock(level, lockId, grabLock, init);
        }

        internal static int NILocks(int level)
        {
            if (Semaphores[level] != null)
                return Semaphores[level].Length;
            return 32;
        }

        internal static ILock Barrier(int level, int lockID)
        {
            ILock bl = new ILock(level, lockID, false, 0);
            isBarrier[bl.level][bl.lockId] = true;
            return bl;
        }

        internal static ILock Barrier(int level, Address gaddr)
        {
            ILock bl = new ILock(level, gaddr, false, 0);
            isBarrier[bl.level][bl.lockId] = true;
            return bl;
        }

        internal void BBDecCntr()
        {
            int theIndex = LookupMyIndex();
            LockInfo li = new LockInfo(level, lockId, true);
            using (new LockAndElevate(LInfoLock))
            {
                li.Next = LInfo[theIndex].First;
                LInfo[theIndex].First = li;
            }
            Semaphore theSema;
            using (new LockAndElevate(SemaphoresLock))
            {
                theSema = Semaphores[level][lockId];
                ++nWaiters[level][lockId];
            }
            if ((IsisSystem.Debug & IsisSystem.LOCKSTATE) != 0)
                Isis.WriteLine("BBDecCntr acquiring semaphore[" + level + "][" + lockId + "]");
            try
            {
                if (theSema != null)
                    while ((IsisSystem.IsisActive || !IsisSystem.IsisWasActive) && !theSema.WaitOne(2500))
                        continue;
            }
            finally
            {
                if (IsisSystem.IsisActive == false && !IsisSystem.shuttingDown) throw new IsisShutdown("Isis Inactive");
                using (new LockAndElevate(SemaphoresLock))
                    if (Semaphores[level][lockId] == theSema)
                        --nWaiters[level][lockId];
                using (new LockAndElevate(LInfoLock))
                    LInfo[theIndex].First = li.Next;
            }
        }

        internal void BBIncCntr()
        {
            if ((IsisSystem.Debug & IsisSystem.LOCKSTATE) != 0)
                Isis.WriteLine("BBIncCntr releasing semaphore[" + level + "][" + lockId + "]");
            try { Semaphores[level][lockId].Release(1); }
            catch { }
        }

        internal void Lock()
        {
            LockIt(true);
        }

        private void LockIt(bool willHold)
        {
            if (myIndex == -1)
                myIndex = LookupMyIndex();
            LockInfo li, nli;
            using (new LockAndElevate(LInfoLock))
            {
                li = LInfo[myIndex].First;
                nli = new LockInfo(level, lockId);
                if (li != null && li.LockLevel >= level)
                {
                    bool cycle = false;

                    if (li.LockLevel > level)
                        cycle = true;
                    else
                    {
                        if (li.LockId == lockId && isBarrier[level][lockId] == false)
                            nli.ReLocked = true;
                        else
                        {
                            if (li.LockId > lockId)
                                cycle = true;
                        }
                    }
                    if (cycle)
                    {
                        // Possible cycle but before crying wolf, double check to make sure this isn't a relock request
                        LockInfo tmp;
                        for (tmp = li.Next; tmp != null; tmp = tmp.Next)
                            if (tmp.LockId == lockId && tmp.LockLevel == level)
                            {
                                nli.ReLocked = true;
                                cycle = false;
                                break;
                            }

                        if (cycle)
                            throw new IsisException("Lock cycle: Thread <" + Thread.CurrentThread.Name + "> index [" + myIndex + "] requested " + level + "/" + lockId + " while holding " + li + "\r\n" + ILock.GetState());
                    }
                }
                nli.WaitingFor = !nli.ReLocked;
                nli.Next = li;
                LInfo[myIndex].First = nli;
            }

            if (nli.ReLocked == false)
            {
                if ((IsisSystem.Debug & IsisSystem.LOCKSTATE) != 0)
                    Isis.WriteLine("Semaphore wait: [" + level + "][" + lockId + "]");

                using (new LockAndElevate(SemaphoresLock))
                    ++nWaiters[level][lockId];
                try
                {
                    Semaphores[level][lockId].WaitOne();
                }
                finally
                {
                    if (IsisSystem.IsisActive == false) throw new IsisShutdown("Isis Inactive");

                    using (new LockAndElevate(SemaphoresLock))
                    {
                        --nWaiters[level][lockId];
                        using (new LockAndElevate(LInfoLock))
                        {
                            nli.WaitingFor = false;
                            if (willHold)
                                ++Holding[level][lockId];
                            else
                            {
                                if (LInfo[myIndex].First == nli)
                                    LInfo[myIndex].First = nli.Next;
                                else
                                    throw new IsisException("Error when unwinding lock-state stack!");
                            }
                        }
                    }
                }
                if ((IsisSystem.Debug & IsisSystem.LOCKSTATE) != 0)
                    Isis.WriteLine("Semaphore acquired: [" + level + "][" + lockId + "]");
            }
            else if (willHold)
                using (new LockAndElevate(SemaphoresLock))
                    ++Holding[level][lockId];
            else if ((IsisSystem.Debug & IsisSystem.LOCKSTATE) != 0)
                Isis.WriteLine("No need to wait (re-acquire): [" + level + "][" + lockId + "]");
        }

        internal static int GetLockId(int level, int hashcode)
        {
            int rval = 0;
            using (new LockAndElevate(LockIdsLock))
            {
                if (LockIds[level] == null)
                    LockIds[level] = new Dictionary<int, IdAndCtr>(100);
                IdAndCtr idc;
                if (LockIds[level].TryGetValue(hashcode, out idc))
                {
                    rval = idc.Id;
                    ++idc.Cntr;
                }
                else
                {
                    rval = 0;
                    if (quickPick != -1)
                    {
                        long b = 1;
                        while ((quickPick & b) != 0)
                        {
                            b <<= 1;
                            ++rval;
                        }
                    }
                    else
                    {
                        bool fnd = true;
                        rval = 64;
                        while (fnd)
                        {
                            fnd = false;
                            foreach (KeyValuePair<int, IdAndCtr> kvp in LockIds[level])
                                if (kvp.Value.Id == rval)
                                {
                                    fnd = true;
                                    ++rval;
                                }
                        }
                    }
                    if (rval < 64)
                        quickPick |= 1L << rval;
                    LockIds[level].Add(hashcode, new IdAndCtr(rval));
                }
            }
            return rval;
        }

        private int LookupMyIndex()
        {
            int theIndex = -1;
            using (new LockAndElevate(LInfoLock))
            {
                int id = Thread.CurrentThread.ManagedThreadId;
                if (Waiting.TryGetValue(id, out theIndex) && theIndex >= 0 && theIndex < LInfo.Length)
                {
                    if (LInfo[theIndex].Tname != Thread.CurrentThread.Name)
                    {
                        // Thread id got recycled by the runtime system
                        if (LInfo[theIndex].First != null)
                        {
                            string list = "";
                            for (LockInfo lio = LInfo[theIndex].First; lio != null; lio = lio.Next)
                                list += " [" + lio.LockLevel + "][" + lio.LockId + "] ";
                            throw new IsisException("ISIS: Thread " + LInfo[theIndex].Tname + " terminated while holding locks {" + list + "}!");
                        }
                        LInfo[theIndex].Tname = Thread.CurrentThread.Name;
                    }
                }
                else
                {
                    // New thread id I haven't seen previously
                    if (next == LILEN)
                    {
                        LILEN <<= 1;
                        Isis.ArrayResize<LockInfoHead>(ref LInfo, LILEN);
                        Isis.ArrayResize<PWaitInfo>(ref ProcessWait, LILEN);
                        Isis.ArrayResize<Semaphore>(ref Barriers, LILEN);
                    }
                    Waiting[id] = theIndex = next++;
                    String name = Thread.CurrentThread.Name;
                    LInfo[theIndex] = new LockInfoHead(name);
                }
            }
            return theIndex;
        }

        internal void Release()
        {
            if (!IsisSystem.IsisActive)
                return;
            using (new LockAndElevate(SemaphoresLock))
            {
                int nw;
                nw = nWaiters[level][lockId];
                if (nw > 0 || Holding[level][lockId] > 0)
                {
                    if (--Holding[level][lockId] < 0)
                        throw new IsisException("Isis lock " + level + "/" + lockId + " released more times than it was locked");
                    if ((IsisSystem.Debug & IsisSystem.LOCKSTATE) != 0)
                        Isis.WriteLine("Lock Release for lock[" + level + "][" + lockId + "]");
                    using (new LockAndElevate(LInfoLock))
                    {
                        LockInfo li = null;
                        if (myIndex != -1 && (li = LInfo[myIndex].First) != null)
                        {
                            LInfo[myIndex].First = li.Next;
                            if (!li.ReLocked)
                                Semaphores[level][lockId].Release(1);
                        }
                        else
                            throw new IsisException("Release: release what?");
                    }
                }
            }
        }

        internal void BarrierWait()
        {
            if ((IsisSystem.Debug & IsisSystem.LOCKSTATE) != 0)
                Isis.WriteLine("Enter BarrierWait: simple case [" + level + "][" + lockId + "]");
            int theIndex = LookupMyIndex();
            using (new LockAndElevate(BarriersLock))
                Barriers[theIndex] = Semaphores[level][lockId];
            LockIt(false);
            using (new LockAndElevate(BarriersLock))
                Barriers[theIndex] = null;
            if ((IsisSystem.Debug & IsisSystem.LOCKSTATE) != 0)
                Isis.WriteLine("After BarrierWait: simple case [" + level + "][" + lockId + "]");
        }

        internal void BarrierWait(Group g)
        {
            BarrierWait(g, g.getLiveMembers());
        }

        internal void BarrierWait(Group g, Address[] waitingFor)
        {
            if ((IsisSystem.Debug & IsisSystem.LOCKSTATE) != 0)
                Isis.WriteLine("Enter BarrierWait: [" + level + "][" + lockId + "] group " + g.gname + ", Waiting for " + Address.VectorToString(waitingFor));
            int theIndex = LookupMyIndex();
            using (new LockAndElevate(BarriersLock))
                Barriers[theIndex] = Semaphores[level][lockId];
            using (new LockAndElevate(ProcessWaitLock))
                ProcessWait[theIndex] = new PWaitInfo(g.gaddr, waitingFor);
            LockIt(false);
            using (new LockAndElevate(ProcessWaitLock))
                ProcessWait[theIndex] = null;
            using (new LockAndElevate(BarriersLock))
                Barriers[theIndex] = null;
            if ((IsisSystem.Debug & IsisSystem.LOCKSTATE) != 0)
                Isis.WriteLine("BarrierWait: [" + level + "][" + lockId + "] Other side of the barrier! Group " + g + ", Waiting for " + Address.VectorToString(waitingFor));
        }

        internal void BarrierRelease(int cnt)
        {
            if ((IsisSystem.Debug & IsisSystem.LOCKSTATE) != 0)
                Isis.WriteLine("Barrier Release: [" + level + "][" + lockId + "]");
            Semaphores[level][lockId].Release(cnt);
        }

        // Unlike the other BarrierWait/Release options, ReleaseAll assumes the Waits are done before the Release!
        internal void BarrierReleaseAll()
        {
            if ((IsisSystem.Debug & IsisSystem.LOCKSTATE) != 0)
                Isis.WriteLine("Barrier Release All: [" + level + "][" + lockId + "]");
            int nw;
            using (new LockAndElevate(SemaphoresLock))
                nw = nWaiters[level][lockId];
            if (nw > 0)
                Semaphores[level][lockId].Release(nw);
        }

        private static Address RIP;

        internal static void NoteFailed(Address gaddr, Address rip)
        {
            if (RIP == rip)
                return;
            RIP = rip;

            using (new LockAndElevate(ProcessWaitLock))
            {
                for (int id = 0; id < ProcessWait.Length; id++)
                {
                    PWaitInfo pwi = ProcessWait[id];
                    int a;
                    if (pwi == null || (pwi.gaddr != null && pwi.gaddr.Equals(gaddr) == false) || Barriers[id] == null)
                        continue;
                    for (a = 0; a < pwi.plist.Length; a++)
                        if (pwi.plist[a].Equals(rip))
                            break;
                    if (a == pwi.plist.Length)
                        continue;
                    if (pwi.plist.Length == 1)
                    {
                        using (new LockAndElevate(BarriersLock))
                            if (Barriers[id] != null)
                            {
                                if ((IsisSystem.Debug & IsisSystem.LOCKSTATE) != 0)
                                    Isis.WriteLine("NoteFail doing a Barrier Release(1) on Semaphore associated with Barrier[" + id + "]");
                                Barriers[id].Release(1);
                            }
                        return;
                    }
                    while (a < pwi.plist.Length - 1)
                    {
                        pwi.plist[a] = pwi.plist[a + 1];
                        a++;
                    }
                }
            }
        }

        public void Dispose()
        {
            Dispose(true);
        }

        private void Dispose(bool disposing)
        {
            lock (this)
            {
                if (!disposed)
                {
                    disposed = true;
                    if (disposing)
                    {
                        // Dispose managed resources.
                        using (new LockAndElevate(LockIdsLock))
                        {
                            if (insideUsing)
                            {
                                // This case arises entirely because of the construct "using(new ILock(...)) something
                                // The lock is grabbed, statement is executed, and the dispose that using then invokes triggers release
                                // of the lock.  We do this exactly once, and only in this specific case
                                //
                                // In particular, it is a (serious) bug to forget to call Release if you have an ILockRef and then lock it
                                // I could check for this, but it would slow the lock subsystem down, so I don't
                                //
                                if ((IsisSystem.Debug & IsisSystem.LOCKSTATE) != 0)
                                    Isis.WriteLine("Disposing [" + level + "][" + lockId + "] (Holding[l,id] = " + Holding[level][lockId] + ")");
                                Release();
                            }
                        }
                    }
                }
            }
        }

        ~ILock()
        {
            Dispose(true);
        }
    }

    /// <summary>
    /// Bounded circular buffer.  By convention, a put(null) will cause any reader thread
    /// to terminate; we use this when disposing of these buffers.  The code can handle
    /// multiple readers but in fact Isis2 only has a single reader thread at a time in the
    /// code paths that use these
    /// </summary>
    public class BoundedBuffer
    {
        internal static List<BoundedBuffer> BBList = new List<BoundedBuffer>();
        internal static LockObject BBListLock = new LockObject(false, "BBListLock");
        internal string name;
        volatile internal bool lockedAgainstPut = false;
        internal int size;
        internal int pNext;
        internal int gNext;
        internal int myLockLevel;
        internal int myPlockId;
        internal int myGlockId;
        volatile internal int fullSlots;
        internal int delayedNotifyCnt;
        internal object[] theBuffer;
        internal LockObject Lock;
        internal ILock puttingLock;
        internal ILock gettingLock;
        internal int LastPushedVID;
        internal int LastDeliveredVID;

        /// <summary>
        /// Constructor for a bounded buffer, uses current thread priority for the locking priority level
        /// </summary>
        /// <param name="s"></param>
        /// <param name="sz"></param>
        /// <param name="lockLevel"></param>
        /// <param name="plockId"></param>
        /// <param name="glockId"></param>
        public BoundedBuffer(string s, int sz, int lockLevel, int plockId, int glockId)
            : this(s, sz, lockLevel, plockId, glockId, Thread.CurrentThread.Priority)
        {
        }

        /// <summary>
        /// Constructor for a bounded buffer, includes desired locking priority level
        /// </summary>
        /// <param name="s"></param>
        /// <param name="sz"></param>
        /// <param name="lockLevel"></param>
        /// <param name="plockId"></param>
        /// <param name="glockId"></param>
        /// <param name="pri"></param>
        public BoundedBuffer(string s, int sz, int lockLevel, int plockId, int glockId, ThreadPriority pri)
        {
            using (new LockAndElevate(BBListLock))
            {
                if (plockId == -1)
                {
                    int nILocks = ILock.NILocks(lockLevel) + 2;
                    bool[] inuse = new bool[nILocks];
                    foreach (BoundedBuffer bb in BBList)
                        if (bb.myLockLevel == lockLevel)
                            inuse[bb.myPlockId] = inuse[bb.myGlockId] = true;
                    for (int lid = 0; lid < nILocks; lid += 2)
                        if (!inuse[lid] && !inuse[lid + 1])
                        {
                            plockId = lid;
                            glockId = lid + 1;
                            break;
                        }
                }
                myLockLevel = lockLevel;
                myPlockId = plockId;
                myGlockId = glockId;
                name = s;
                // Leave room for one null, to deal with closing the buffer
                size = sz + 1;
                theBuffer = new object[size];
                // When putting a null, won't wait
                puttingLock = ILock.IlockResetAndRef(lockLevel, plockId, false, size);
                gettingLock = ILock.IlockResetAndRef(lockLevel, glockId, false, 0);
                BBList.Add(this);
            }
            Lock = new LockObject(name + "/BoundedBuffer.Lock", pri);
        }

        /// <summary>
        /// Removes a buffer from the list that will be closed down if Shutdown is called
        /// </summary>
        /// <param name="bb"></param>
        public static void unregister(BoundedBuffer bb)
        {
            using (new LockAndElevate(BBListLock))
                BBList.Remove(bb);
            bb.put(null, true);
        }

        /// <summary>
        /// Used to close down the entire buffering subsystem
        /// </summary>
        public static void ShutDown()
        {
            using (new LockAndElevate(BBListLock))
                foreach (BoundedBuffer bb in BBList)
                    bb.put(null);
        }

        /// <summary>
        /// Returns a string summarizing the buffer state
        /// </summary>
        /// <returns></returns>
        public static string GetState()
        {
            string bs = "BOUNDED BUFFERS:\r\n";
            using (new LockAndElevate(BBListLock))
                foreach (BoundedBuffer bb in BBList)
                    using (new LockAndElevate(bb.Lock))
                    {
                        bs += "  <" + bb.name + ">  size=" + (bb.size - 1) + " (" + (bb.fullSlots < bb.size ? bb.fullSlots.ToString() : "all") + " full)" + (bb.lockedAgainstPut ? " locked" : "") +
                            ", GetLock=" + ILock.PLock(bb.gettingLock) + ", PutLock=" + ILock.PLock(bb.puttingLock);
                        if (bb.delayedNotifyCnt > 0)
                            bs += ", Delayed Notify Count=" + bb.delayedNotifyCnt;
                        bs += "\r\n";
                    }
            return bs;
        }

        /// <summary>
        /// Tests to see if the bb has enough room to hold howMany additional objects
        /// </summary>
        /// <param name="howMany"></param>
        /// <returns></returns>
        public bool putWillBlock(int howMany)
        {
            using (new LockAndElevate(Lock))
                return size < (fullSlots + howMany);
        }

        /// <summary>
        /// Returns the number of full slots in the bounded buffer
        /// </summary>
        /// <param name="bb">Buffer to check</param>
        /// <returns>Number of full slots</returns>
        public static int FullSlots(BoundedBuffer bb)
        {
            if (bb == null)
                return 0;
            return bb.FullSlots();
        }

        /// <summary>
        /// Returns the number of full slots in the bounded buffer
        /// </summary>
        /// <returns>Number of full slots</returns>
        public int FullSlots()
        {
            using (new LockAndElevate(Lock))
                return fullSlots;
        }

        /// <summary>
        ///  Puts an object in the buffer if there is room.  If not, waits
        /// </summary>
        /// <param name="o">object to store</param>
        public void put(object o)
        {
            put(o, false);
        }

        /// <summary>
        /// Puts an object in the buffer if there is room.  If not, waits if nonBlocking=false, returns (with no error indication) if nonBlocking=true
        /// </summary>
        /// <param name="o">object to store</param>
        /// <param name="nonBlocking">NonBlocking mode indication</param>
        public void put(object o, bool nonBlocking)
        {
            using (new LockAndElevate(Lock))
                if (nonBlocking && fullSlots == size)
                    return;
            if (o != null)
                puttingLock.BBDecCntr();
            using (new LockAndElevate(Lock))
            {
                if (lockedAgainstPut)
                {
                    puttingLock.BBIncCntr();
                    return;
                }
                if (o != null && LastPushedVID > 0)
                {
                    Type t = o.GetType();
                    if (t.Equals(typeof(View)))
                        LastPushedVID = ((View)o).viewid;
                    else if (t.Equals(typeof(Msg)))
                    {
                        Msg m = (Msg)o;
                        if (m.vid >= 0 && m.vid != LastPushedVID)
                            throw new IsisException("put: putting a message " + m.sender + "::" + m.vid + ":" + m.msgid + " when " + name + " was using viewid=" + LastPushedVID);
                    }
                    else if (t.Equals(typeof(List<Msg>)))
                        foreach (Msg m in (List<Msg>)o)
                            if (m.vid >= 0 && m.vid != LastPushedVID)
                                throw new IsisException("put/List: putting a message " + m.sender + "::" + m.vid + ":" + m.msgid + " when " + name + " was using viewid=" + LastPushedVID);
                }
                theBuffer[(pNext++) % size] = o;
                if (++fullSlots > size && o != null && !lockedAgainstPut)
                    Isis.WriteLine("WARNING... BB:put<" + name + ">: fullslots overflow");
                if (o == null)
                {
                    lockedAgainstPut = true;
                    puttingLock.BBIncCntr();
                }
            }
            gettingLock.BBIncCntr();
        }

        /// <summary>
        /// Used (only) from SafeSend and OrderedSend; puts an object on FRONT of the circular buffer.  The idea is to replace the SETORDER message
        /// that was just delivered (previous front of the circular buffer) with a list of the ordered messages that it released for delivery, and in
        /// the order it specified (obviously).  Then those will be delivered one by one.  (Since none of them will be a SETORDER message, all of those
        /// will have been delivered by the time the next SETORDER message is done, hence we don't run the risk of two concurrent putFront events occuring)
        /// </summary>
        /// <param name="o">An object to place in the buffer</param>
        public void putFront(object o)
        {
            puttingLock.BBDecCntr();
            using (new LockAndElevate(Lock))
            {
                if (lockedAgainstPut)
                {
                    puttingLock.BBIncCntr();
                    return;
                }
                if (--gNext < 0)
                    gNext += size;
                if (o != null && LastDeliveredVID != 0)
                {
                    List<Msg> ml = (List<Msg>)o;
                    foreach (Msg m in ml)
                        if (m.vid >= 0 && m.vid != LastDeliveredVID)
                            throw new IsisException("putFront: putting a message " + m.sender + "::" + m.vid + ":" + m.msgid + " when " + name + " was using viewid=" + LastDeliveredVID);
                }
                theBuffer[gNext % size] = o;
                if (++fullSlots > size && o != null && !lockedAgainstPut)
                    Isis.WriteLine("WARNING... BB:putFront<" + name + ">: fullslots overflow");
            }
            gettingLock.BBIncCntr();
        }

        /// <summary>
        /// Get the next object in the buffer
        /// </summary>
        /// <returns>Object found.  Blocks if the bb is empty.</returns>
        public object get()
        {
            object o;
            bool delayedNotify = false;
            int delayednotifies = 0;
            gettingLock.BBDecCntr();
            using (new LockAndElevate(Lock))
            {
                int idx = (gNext++) % size;
                o = theBuffer[idx];
                if (o != null)
                {
                    Type t = o.GetType();
                    if (t.Equals(typeof(View)))
                        LastDeliveredVID = ((View)o).viewid;
                }
                if (fullSlots <= 0)
                {
                    if (!IsisSystem.IsisActive && IsisSystem.IsisWasActive)
                        return null;
                    if (!lockedAgainstPut)
                        Isis.WriteLine("WARNING... BB:get<" + name + ">: fullslots underflow with gettinglock=" + gettingLock.lockId);
                    for (int i = 0; i < 10; i++)
                        Isis.WriteLine("**********************************************************************************************************************");
                    return null;
                }
                --fullSlots;
                theBuffer[idx] = null;
                if (fullSlots > (size * 3) / 4 && o != null && myLockLevel != ILock.LLTCP)
                {
                    delayedNotify = true;
                    ++delayedNotifyCnt;
                }
                else
                {
                    delayednotifies = delayedNotifyCnt + 1;
                    delayedNotifyCnt = 0;
                }
            }
            while (!delayedNotify && !lockedAgainstPut && --delayednotifies >= 0)
                puttingLock.BBIncCntr();
            return o;
        }
    }

    internal class LockingDependencyOrder
    {
        internal LockObject otherLock;
        internal string stack;

        internal LockingDependencyOrder(LockObject lo, string st)
        {
            otherLock = lo; stack = st;
        }
    }

    internal class LockObject
    {
        internal object Info = new object();
        internal ThreadPriority AssociatedPriority;
#if TRACKLOCKINFO
        internal int lockHolderId = 0;
        internal bool TrackOrder = true;
        internal string lastlockedInfo = "???";
#endif
        internal bool isLocked = false;
        internal Thread lockHolder = null;
        internal string lockedBy = "???";
        internal string Name = "???";
        internal long UseCount;
        internal long LockedAt;
        internal List<Thread> WantsMe = new List<Thread>();
        internal List<LockingDependencyOrder> DependencyOrder = new List<LockingDependencyOrder>();
        internal static List<LockObject> LockObjList = new List<LockObject>();

        internal LockObject(bool TrackOrder, string name) : this(TrackOrder, name, Thread.CurrentThread.Priority) { }
        internal LockObject(string name) : this(true, name, Thread.CurrentThread.Priority) { }
        internal LockObject(string name, ThreadPriority usePriority) : this(true, name, usePriority) { }
        internal LockObject(bool TrackOrder, string name, ThreadPriority usePriority)
        {
            Name = name;
            AssociatedPriority = usePriority;
#if TRACKLOCKINFO
            lock (LockObjList)
                LockObjList.Add(this);
            this.TrackOrder = TrackOrder;
#endif // TRACKLOCKINFO
        }
        internal LockObject()
        {
            UseCount = 0;
            LockedAt = 0;
        }

        ~LockObject()
        {
#if TRACKLOCKINFO
            lock (LockObjList)
                LockObjList.Remove(this);
#endif // TRACKLOCKINFO
        }

        internal static string GetState()
        {
            return GetState(false);
        }

        internal static string GetState(bool includeCallStack)
        {
#if TRACKLOCKINFO
            string s = "LOCKED OBJECTS:\r\n";
            if (includeCallStack)
                s += "    .... dump was requested by " + threadInfo() + "\r\n";
            List<LockObject> LockObjListClone = new List<LockObject>();
            lock (LockObjList)
                foreach (LockObject lo in LockObjList)
                    LockObjListClone.Add(lo);
            foreach (LockObject lo in LockObjListClone)
                lock (lo.Info)
                {
                    if (!lo.isLocked)
                        continue;
                    string wantedBy = "", depinfo = " ", lockedby = "";
                    foreach (LockingDependencyOrder ldo in lo.DependencyOrder)
                        depinfo += ldo.otherLock.Name + " ";
                    if (lo.lockHolder != null)
                        lockedby = "locked by Thread[" + lo.lockHolder.Name + "] for " + Isis.TimeToString(Isis.NOW() - lo.LockedAt) + (wantedBy.Length > 0 ? ("; Wanted by " + wantedBy) : "");
                    foreach (Thread t in lo.WantsMe)
                        wantedBy += "Thread[" + t.ManagedThreadId + "] (" + (t.Name ?? "no name") + " ";
                    s += "   <" + lo.Name + ">: UseCnt=" + lo.UseCount + "; " + lockedby + "\r\n";
                    if (includeCallStack)
                        s += "    .... call-stack that acquired lock = " + lo.lockedBy + "\r\n";
                }
#else // TRACKLOCKINFO
            string s = "LOCKED OBJECTS: NOT TRACKING DETAILED USE\r\n";
#endif // TRACKLOCKINFO
            return s;
        }

        internal static string LocksIHold()
        {
#if TRACKLOCKINFO
            string lockList = " ";
            lock (LockObjList)
                foreach (LockObject lo in LockObjList)
                    if (lo.lockHolder == Thread.CurrentThread)
                        lockList += lo.Name + " ";
            return lockList;
#else // TRACKLOCKINFO
            return "not tracked";
#endif // TRACKLOCKINFO
        }

        internal static void ThreadIsTerminating()
        {
            List<LockObject> toUnlock = new List<LockObject>();
            lock (LockObjList)
                foreach (LockObject lo in LockObjList)
                    lock (lo.Info)
                        if (lo.lockHolder == Thread.CurrentThread)
                            toUnlock.Add(lo);
            foreach (LockObject lo in toUnlock)
                lock (lo.Info)
                    if (lo.lockHolder == Thread.CurrentThread)
                    {
                        if (lo.isLocked)
                            System.Threading.Monitor.Exit(lo);
                        lo.lockHolder = null;
                        lo.lockedBy = null;
                        lo.isLocked = false;
                    }
            if (IsisSystem.IsisActive)
                foreach (LockObject lo in toUnlock)
                    Isis.WriteLine("************ ThreadIsTerminating unlocked <" + lo.Name + "> on behalf of " + Thread.CurrentThread.Name);
        }

        internal static string threadInfo()
        {
            return "Thread [" + Thread.CurrentThread.ManagedThreadId + (Thread.CurrentThread.Name == null ? "" : "(" + Thread.CurrentThread.Name) + ")]";
        }
    }

    internal class LockAndElevate : IDisposable
    {
        internal LockObject LockedObject;
        internal ThreadPriority myPriority;
        internal int lockedCnt = 0;
        internal bool disposed = false;
        internal bool elevated = false;
        internal static bool disabled = false;

        internal LockAndElevate(LockObject Lock) : this(Lock, false) { }

        internal LockAndElevate(LockObject Lock, bool inhibitWARNINGs)
        {
            if (disabled)
                return;
#if TRACKLOCKINFO
            string lb = LockObject.threadInfo();
#endif // TRACKLOCKINFO
            lock (Lock.Info)
            {
#if TRACKLOCKINFO
                if (Lock.lockHolder == Thread.CurrentThread)
                {
                    lock (this)
                        ++lockedCnt;
                    return;
                }
#endif // TRACKLOCKINFO
                LockedObject = Lock;
                myPriority = Thread.CurrentThread.Priority;
                if (myPriority > Lock.AssociatedPriority)
                {
                    if (Isis.NOW() > 15000 && Lock.isLocked)
                        Isis.WriteLine("WARNING: Elevating priority on Lock <" + Lock.Name + "> from " + Lock.AssociatedPriority +
                            " to " + myPriority + " while held by " + Lock.lockedBy);
                    Lock.AssociatedPriority = myPriority;
                }
                else
                    Thread.CurrentThread.Priority = Lock.AssociatedPriority;
            }
#if TRACKLOCKINFO
            if (!inhibitWARNINGs)
                noteAssociation(Thread.CurrentThread.ManagedThreadId, Lock);
#endif // TRACKLOCKINFO
            bool lockTaken = false;
            long lcount = 0;
            int tout = Isis.ISIS_DEFAULTTIMEOUT * 2 / 5, time;
            for (time = 0; !lockTaken && (IsisSystem.IsisActive || !IsisSystem.IsisWasActive) && time < 10; time++)
            {
                System.Threading.Monitor.TryEnter(LockedObject, tout, ref lockTaken);
                if (!lockTaken)
                {
                    if (!IsisSystem.IsisActive && IsisSystem.IsisWasActive && time == 4)
                        throw new IsisShutdown("Isis is shutting down, LockAndElevate timeout");
                    string name = "";
                    lock (Lock.Info)
                    {
                        bool newHolder = false;
                        if (lcount != Lock.UseCount)
                        {
                            newHolder = (lcount != 0);
                            lcount = Lock.UseCount;
                        }
                        if (newHolder)
                            name = " ** lock holder changed since last time I looked ** ";
                        else if (!Lock.isLocked)
#if TRACKLOCKINFO
                            name = " ** lock isn't locked right now! Usecnt = " + Lock.UseCount + " ** ";
#else // TRACKLOCKINFO
                            name = " ** not tracking lockstate (recompile with TRACKLOCKINFO to enable tracking) ** ";
#endif // TRACKLOCKINFO
                        else if (Lock.lockHolder != null && Lock.lockHolder.Name != null)
                            name = " (lock held by " + Lock.lockHolder.Name + ")";
                    }
                    if ((IsisSystem.Debug & IsisSystem.WARNIFSLOW) != 0)
                        Isis.WriteLine("WARNING: " + LockObject.threadInfo() + " has waited for " + Lock.Name + " for " + ((time + 1) * tout / 1000) + " secs" + name);
                }
            }
            if (!lockTaken)
            {
                string tname = "<thread has no name>";
                if (Thread.CurrentThread.Name != null)
                    tname = Thread.CurrentThread.Name;
                Isis.WriteLine("Unable to acquire " + Lock.Name + " for " + tname + " in " + LockObject.GetState());
                string name = "";
                lock (Lock.Info)
                {
                    if (!Lock.isLocked)
                        name = " ** [unknown thread] ** Lock usecnt = " + Lock.UseCount + " ** ";
                    else if (Lock.lockHolder != null && Lock.lockHolder.Name != null)
                        name = " (lock held by " + Lock.lockHolder.Name + ")";
                }
                new Thread(delegate()
                {
                    try
                    {
                        Isis.WriteLine("Lock<" + name + "> request failed in state: " + LockObject.GetState());
                    }
                    catch (IsisShutdown) { IsisSystem.CheckLocksHeld(); }
                    IsisSystem.ThreadTerminationMagic();
                }).Start();
                if (!IsisSystem.IsisActive && IsisSystem.IsisWasActive)
                    throw new IsisShutdown("");
                throw new IsisException("[" + Thread.CurrentThread.Name + "@" + Isis.TimeToString(Isis.NOW()) + "]: Isis was unable to acquire " + Lock.Name +
                    " after waiting " + ((time + 1) * tout / 1000) + " secs" + name + "\r\nFailed in lock-state " + LockObject.GetState(true));
            }
#if TRACKLOCKINFO
            lock (Lock.Info)
            {
                Lock.isLocked = true;
                Lock.lockedBy = lb;
                Lock.LockedAt = Isis.NOW();
                Lock.lockHolder = Thread.CurrentThread;
                Lock.lockHolderId = Thread.CurrentThread.ManagedThreadId;
                ++Lock.UseCount;
            }
#else
            lock (Lock.Info)
                Lock.lockHolder = Thread.CurrentThread;
#endif // TRACKLOCKINFO
            elevated = true;
        }

        internal static void Disable(bool onOff)
        {
            disabled = onOff;
        }

        internal static List<LockObject>[] LockList = new List<LockObject>[1000];
        internal static object LockLock = new object();

        internal static void noteAssociation(int id, LockObject lo)
        {
            if (!IsisSystem.IsisActive && IsisSystem.IsisWasActive)
                return;
#if TRACKLOCKINGORDER
            if (!lo.TrackOrder || (id < 0 && id >= LockList.Length))
                return;
            List<LockObject> theList = new List<LockObject>();
            lock (LockLock)
            {
                if (LockList[id] == null)
                {
                    LockList[id] = new List<LockObject>();
                    LockList[id].Add(lo);
                    return;
                }
                LockList[id].Add(lo);
                foreach (LockObject theLo in LockList[id])
                    theList.Add(theLo);
            }
            foreach (LockObject otherLo in theList)
                if (otherLo != lo && otherLo.TrackOrder)
                {
                    bool fnd = false;
                    lock (otherLo.Info)
                        if (otherLo.isLocked && otherLo.lockHolder == Thread.CurrentThread)
                        {
                            fnd = true;
                            foreach (LockingDependencyOrder ldo in otherLo.DependencyOrder)
                                if (ldo.otherLock == lo)
                                    Isis.WriteLine("WARNING: sometimes  <" + lo.Name + "> is locked first, then <" + otherLo.Name + ">, but locked <" + otherLo.Name + "> first and then <" + lo.Name + ">!\r\n" +
                                        "      This thread called at [" + Isis.ExtractStackTrace() + "]\r\n    Saw other dependency at [ " + ldo.stack + "]");
                        }
                    if (!fnd)
                        continue;
                    fnd = false;
                    lock (lo.Info)
                    {
                        foreach (LockingDependencyOrder ldo in lo.DependencyOrder)
                            if (ldo.otherLock == otherLo)
                            {
                                fnd = true;
                                break;
                            }
                        if (!fnd)
                            lo.DependencyOrder.Add(new LockingDependencyOrder(otherLo, Isis.ExtractStackTrace()));
                    }
                }
#endif // TRACKLOCKINGORDER
        }

        void IDisposable.Dispose()
        {
            Dispose(true);
        }

        internal void Dispose(bool disposing)
        {
            lock (this)
            {
                if (!disposed)
                {
                    disposed = true;
                    if (lockedCnt > 0 || (IsisSystem.IsisAlreadyRan && !IsisSystem.IsisActive))
                    {
                        --lockedCnt;
                        return;
                    }
                    if (disposing)
                    {
                        if (LockedObject != null && LockedObject.Info != null && LockedObject.lockedBy != null)
                        {
                            string tname = "unknown";
                            Thread lockHolder;
                            lock (LockedObject.Info)
                            {
                                lockHolder = LockedObject.lockHolder;
                                if (lockHolder != null)
                                    tname = lockHolder.Name;
#if TRACKLOCKINFO
                                if (!LockedObject.isLocked)
                                    Isis.WriteLine("WARNING: In Dispose for " + LockedObject.Name + ", yet isLocked = false");
                                LockedObject.lockedBy += " *** In dispose, updating lock info";
                                if (LockedObject.TrackOrder)
                                {
                                    lock (LockLock)
                                    {
                                        int id = LockedObject.lockHolderId;
                                        if (id > 0 && id <= LockList.Length)
                                        {
                                            if (LockList[id] != null)
                                            {
                                                if (LockList[id].Remove(LockedObject) == false)
                                                    Isis.WriteLine("WARNING: Surprised not to find <" + LockedObject.Name + "> on LockList[" + id + "] in LockObject.Dispose!");
                                                if (LockList[id].Count() == 0)
                                                    LockList[id] = null;
                                            }
                                        }
                                    }
                                }
                                Thread ht = LockedObject.lockHolder;
                                if (ht == null)
                                    Isis.WriteLine("WARNING: Dispose for Lock " + LockedObject.Name + " but lockHolder=null!");
                                LockedObject.lastlockedInfo = "From " + Isis.TimeToString(LockedObject.LockedAt) + " to " + Isis.TimeToString(Isis.NOW()) + " was locked by " + (ht.Name ?? "unnamed thread");
                                LockedObject.lockedBy = "not locked";
                                LockedObject.lockHolder = null;
                                LockedObject.isLocked = false;
#endif // TRACKLOCKINFO
                            }
                            if (lockHolder != null && lockHolder.Equals(Thread.CurrentThread))
                                try
                                {
                                    System.Threading.Monitor.Exit(LockedObject);
                                }
                                catch (Exception e)
                                {
                                    string why = e.Message;
                                    throw new IsisException("System.Threading.Monitor.Exit threw " + why + " on LockedObject " + LockedObject.Name);
                                }
                            else if (IsisSystem.IsisActive && !Thread.CurrentThread.Name.Equals("Finalizer"))
                                throw new IsisException("Dispose: called for " + LockedObject.Name + " from a thread other than the one that acquired the lock (current thread is < " +
                                    Thread.CurrentThread.Name + "> should be <" + tname + ">)");
                        }
                    }
                    LockedObject = null;
                    Thread.CurrentThread.Priority = myPriority;
                }
            }
        }

        ~LockAndElevate()
        {
            Dispose(true);
        }
    }

}